{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "import configparser\n",
    "\n",
    "import numpy as np\n",
    "import sklearn.linear_model\n",
    "import sklearn.metrics\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from solml import load, cnn\n",
    "import split\n",
    "import get_info\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read('../config.ini')\n",
    "models_filename = config['main']['models']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_ids, val_ids, test_ids, train_labels, val_labels, test_labels = split.get_sets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bounding_boxes = get_info.get_bounding_box(train_ids)\n",
    "cnn_train = cnn.get_cnn_features_list(train_ids, bounding_boxes)\n",
    "bounding_boxes = get_info.get_bounding_box(val_ids)\n",
    "cnn_val = cnn.get_cnn_features_list(val_ids, bounding_boxes)\n",
    "bounding_boxes = get_info.get_bounding_box(test_ids)\n",
    "cnn_test = cnn.get_cnn_features_list(test_ids, bounding_boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class_weight=None # or 'balanced'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR took 0.679146 seconds.\n",
      "LR took 0.914404 seconds.\n",
      "LR took 1.036362 seconds.\n",
      "LR took 1.586230 seconds.\n",
      "LR took 2.500262 seconds.\n",
      "LR took 3.769882 seconds.\n",
      "LR took 8.144414 seconds.\n",
      "LR took 16.307713 seconds.\n",
      "LR took 37.330177 seconds.\n",
      "LR took 60.086904 seconds.\n",
      "LR took 63.062312 seconds.\n",
      "LR took 65.215640 seconds.\n",
      "LR took 61.077338 seconds.\n",
      "LR took 62.773758 seconds.\n",
      "LR took 61.582564 seconds.\n"
     ]
    }
   ],
   "source": [
    "n_C = 15\n",
    "C_list = np.logspace(-10, 3, n_C)\n",
    "errors = np.zeros(n_C)\n",
    "for i, C in enumerate(C_list):\n",
    "    model = sklearn.linear_model.LogisticRegression(penalty='l2', C=C, class_weight=class_weight)\n",
    "    t = time.time()\n",
    "    model.fit(cnn_train, train_labels)\n",
    "    print(\"LR took %f seconds.\"%(time.time()-t))\n",
    "    predictions = model.predict(cnn_val)\n",
    "    errors[i] = (val_labels!=predictions).sum()/len(val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH9FJREFUeJzt3Xt8XHWd//HXJ/dkkqaXmZY2SZtJW4EWamnDRfGCF2wF\naV3LIrq7guIPUar+Vh8q7M8VhN/+frI+Huyu/lgRlRVd5bKtK1G7ILgPFMVKU9oiLRTatLQpvaRN\n2/SSez6/PzIp07Qlk3aSMzPn/Xw88uicM+dM3xPCO6fnfOd8zd0REZFwyAs6gIiIjB6VvohIiKj0\nRURCRKUvIhIiKn0RkRBR6YuIhIhKX0QkRFT6IiIhotIXEQkRlb6ISIgUBB1gsGg06rW1tUHHEBHJ\nKqtXr97r7rGhtsu40q+traWxsTHoGCIiWcXMXk1lO53eEREJEZW+iEiIqPRFREJEpS8iEiIqfRGR\nEFHpi4iESMYN2TxdnT29/O7lvUHHGJaJFcXMqa7EzIKOIiIhkTOlf7ijh//xo+wb3x+PRlgyr4q/\nmFdN1djSoOOISI6zTJsYvb6+3k/nw1ndvX1s3HVoBBKNnA0721i+upk/bWnFDN46fQJXz69mweyz\nKCvKmd/HIjIKzGy1u9cPuV2ulH4227bvKD9b08zy55rZ3tpOpCifK+dM5ur5NVxYO06nf0RkSCr9\nLNTX5zy7tZXlq5v51Z93crSrl6njy1gyr5oPzauiZnxZ0BFFJEOp9LPc0a4eHnthF8tWN/PM5n0A\nXFI3niXzqrni/MlEinX6R0Rep9LPIc37j/Kfz+1g2XPNvLrvKGVF+bz/vMksmV/FJfEJ5OXp9I9I\n2Kn0c5C7s/rV/Sxb3cwvn9/J4c4eqsaWsmReFUvmVzNtQiToiCISEJV+jmvv6uXXG/pP//x+017c\n4aLa8SyZX8XiuVWUFOYHHVFERlGqpZ/SJ3LNbKGZbTSzTWZ2y0mev97MWsxsbeLrk0nP9Satbxje\n25BTKS3KZ/HcKn58w8X84Svv5ksLzmbv4U6+svzPfOO/Xgo6nohkqCGvBppZPnAPcDnQDKwyswZ3\n3zBo04fdfelJXqLd3eeeeVQ5lSljS7n5XTP4zGXT+cj3VrJm+4GgI4lIhkrlSP8iYJO7N7l7F/AQ\nsHhkY8npMDNmTqxgS8thMu20nYhkhlRKvwrYnrTcnFg32BIze97MlplZTdL6EjNrNLOVZvbBMwkr\nQ4tHI7R19LDvSFfQUUQkA6XrLpu/AGrdfQ7wBPBA0nPTEhcXPgr8s5lNH7yzmd2Y+MXQ2NLSkqZI\n4RSP9Y/g2bL3SMBJRCQTpVL6O4DkI/fqxLpj3H2fu3cmFr8PzE96bkfizybgKeCCwX+Bu9/n7vXu\nXh+LDTmZu7yB6dFyAJpaDgecREQyUSqlvwqYaWZxMysCrgWOG4VjZpOTFhcBLybWjzOz4sTjKHAp\nMPgCsKRR1bhSivLzaNKRvoicxJCjd9y9x8yWAo8D+cD97r7ezO4AGt29AficmS0CeoBW4PrE7ucC\n3zWzPvp/wXzjJKN+JI3y84xpE8poalHpi8iJUrqBi7uvAFYMWve1pMe3AreeZL9ngPPPMKMMUzwa\n0ZG+iJyUpkvMQXWxcl7dd4Se3r6go4hIhlHp56C6aITuXmfHgfago4hIhlHp56C6xLBNneIRkcFU\n+jkoHk2Uvi7misggKv0cND5SRGVpIVv2aqy+iBxPpZ+DzKx/BI+O9EVkEJV+jqqLRnQrBhE5gUo/\nR9XFIuw82MHRrp6go4hIBlHp56h44h48OtoXkWQq/Rx1bNimzuuLSBKVfo6qnaBbLIvIiVT6Oaq0\nKJ+qsaUqfRE5jko/h/UP29RYfRF5nUo/h9XF+u+2qflyRWSASj+HxaMRDnX0sPew5ssVkX4q/Rw2\ncA8endcXkQEq/Rw2Pab5ckXkeCr9HDZlbClFBXk60heRY1T6OSw/z6idUMZmfUBLRBJU+jkuHo3o\nFssicoxKP8fVxcrZ1npU8+WKCKDSz3nxxHy5zfs1X66IqPRz3vSYhm2KyOtU+jlu4BbLmzVsU0RI\nsfTNbKGZbTSzTWZ2y0mev97MWsxsbeLrk0nPXWdmryS+rktneBna+EgRY8sKdaQvIgAUDLWBmeUD\n9wCXA83AKjNrcPcNgzZ92N2XDtp3PHAbUA84sDqx7/60pJeUaL5cERmQypH+RcAmd29y9y7gIWBx\niq+/AHjC3VsTRf8EsPD0osrpimu+XBFJSKX0q4DtScvNiXWDLTGz581smZnVDHNfGUHTY+Xsauvg\nSKfmyxUJu3RdyP0FUOvuc+g/mn9gODub2Y1m1mhmjS0tLWmKJAN04zURGZBK6e8AapKWqxPrjnH3\nfe7emVj8PjA/1X0T+9/n7vXuXh+LxVLNLik6Nl+uSl8k9FIp/VXATDOLm1kRcC3QkLyBmU1OWlwE\nvJh4/DjwPjMbZ2bjgPcl1skoqp0QwQy26GKuSOgNOXrH3XvMbCn9ZZ0P3O/u683sDqDR3RuAz5nZ\nIqAHaAWuT+zbamZ30v+LA+AOd28dgfchb6CkMJ8plaW6B4+IDF36AO6+AlgxaN3Xkh7fCtx6in3v\nB+4/g4ySBgNTJ4pIuOkTuSFRF42wpUXz5YqEnUo/JOLRCIc6e2g53Dn0xiKSs1T6IVGXmDpRF3NF\nwk2lHxIDY/V1Xl8k3FT6IaH5ckUEVPqhkZ9nxCdEaNItlkVCTaUfIvGohm2KhJ1KP0TqYhG27dN8\nuSJhptIPkXg0Qk+fs13z5YqElko/RI4N29TtGERCS6UfInUDwzY1Vl8ktFT6ITIuUsS4skJdzBUJ\nMZV+yPTPl6vTOyJhpdIPmXi0XB/QEgkxlX7I1MUi7G7r5LDmyxUJJZV+yAxczN2qo32RUFLph8zA\nsM3NOq8vEkoq/ZCZNqGsf75cHemLhJJKP2RKCvOpGluq0hcJKZV+CPUP21Tpi4SRSj+Epsf6h21q\nvlyR8FHph1A8GuFwZw8thzRfrkjYqPRDqC6mqRNFwkqlH0Jx3XhNJLRU+iE0pbKU4oI83WJZJIRS\nKn0zW2hmG81sk5nd8gbbLTEzN7P6xHKtmbWb2drE173pCi6nLy/PNIJHJKQKhtrAzPKBe4DLgWZg\nlZk1uPuGQdtVAJ8H/jToJTa7+9w05ZU0iUcjbNx1KOgYIjLKUjnSvwjY5O5N7t4FPAQsPsl2dwJ3\nAR1pzCcjpC4WYVvrUbo1X65IqKRS+lXA9qTl5sS6Y8xsHlDj7r86yf5xM1tjZr81s7efflRJp3i0\nvH++3NajQUcRkVF0xhdyzSwPuBv44kme3glMdfcLgC8APzWzMSd5jRvNrNHMGltaWs40kqRgYNim\nbscgEi6plP4OoCZpuTqxbkAFcB7wlJltBS4BGsys3t073X0fgLuvBjYDbxr8F7j7fe5e7+71sVjs\n9N6JDIvmyxUJp1RKfxUw08ziZlYEXAs0DDzp7gfdPerute5eC6wEFrl7o5nFEheCMbM6YCbQlPZ3\nIcM2tqyI8ZEifUBLJGSGHL3j7j1mthR4HMgH7nf39WZ2B9Do7g1vsPs7gDvMrBvoA25y99Z0BJcz\np/lyRcJnyNIHcPcVwIpB6752im0vS3q8HFh+BvlkBMWjEX73sq6hiISJPpEbYnWxCHsOdXKoozvo\nKCIySlT6Ifb6fLkatikSFir9EBuYL7dJ9+ARCQ2VfohNHd8/X66GbYqEh0o/xEoK86kep/lyRcJE\npR9y8Wi5Tu+IhIhKP+TqohG2tGi+XJGwUOmHXF0swpGuXvZovlyRUFDph1xdNDGCRxdzRUJBpR9y\n8WOTpOu8vkgYqPRDbvKYEkoK89iiI32RUFDph1xenlE7IaK7bYqEhEpfqItFNFZfJCRU+kJdtFzz\n5YqEhEpfiEcj9PY52zRfrkjOU+nL6/Pl6mKuSM5T6cvrY/U1bFMk56n0hcqyQiZEinQxVyQEVPoC\n9J/X36zTOyI5T6UvQH/p60hfJPep9AXon0WrRfPliuQ8lb4A/Uf6gI72RXKcSl8AmB5T6YuEgUpf\nAJg6oYw8QxdzRXJcSqVvZgvNbKOZbTKzW95guyVm5mZWn7Tu1sR+G81sQTpCS/oVF+RTPa5MR/oi\nOa5gqA3MLB+4B7gcaAZWmVmDu28YtF0F8HngT0nrZgHXArOBKcCTZvYmd+9N31uQdIlHIzS16ANa\nIrkslSP9i4BN7t7k7l3AQ8Dik2x3J3AX0JG0bjHwkLt3uvsWYFPi9SQDDdxtU/PliuSuVEq/Ctie\ntNycWHeMmc0Datz9V8PdVzJHXTTC0a5edrdpvlyRXHXGF3LNLA+4G/jiGbzGjWbWaGaNLS0tZxpJ\nTlNdTPfgEcl1qZT+DqAmabk6sW5ABXAe8JSZbQUuARoSF3OH2hcAd7/P3evdvT4Wiw3vHUjaDIzV\n1yTpIrkrldJfBcw0s7iZFdF/YbZh4El3P+juUXevdfdaYCWwyN0bE9tda2bFZhYHZgLPpv1dSFqc\nNTBfrkbwiOSsIUfvuHuPmS0FHgfygfvdfb2Z3QE0unvDG+y73sweATYAPcDNGrmTufLyjHi0XCN4\nRHLYkKUP4O4rgBWD1n3tFNteNmj5H4B/OM18MsrqohHWv3Yw6BgiMkL0iVw5Tl0swvb97XT1aL5c\nkVyk0pfjaL5ckdym0pfjDAzb1MVckdyk0pfjvD5sUxdzRXKRSl+OU1laSLRc8+WK5CqVvpyg/8Zr\nKn2RXKTSlxPEoxGadKQvkpNU+nKCulg5ew930qb5ckVyjkpfTnBsvlyd4hHJOSp9OYHmyxXJXSp9\nOUHN+P75cjVsUyT3qPTlBMUF+dSML9PFXJEcpNKXk9KwTZHcpNKXk6qLlmu+XJEcpNKXk4rHIrR3\n97KrrWPojUUka6j05aSma9imSE5S6ctJxRPDNjfrYq5ITlHpy0lNqiihtDBfR/oiOUalLyfVP19u\nhC17NVZfJJeo9OWU4jHdeE0k16j05ZSmRyNsbz2q+XJFcohKX04pHovQ57CtVUf7IrlCpS+nVBft\nny9Xn8wVyR0FQQeQzDUwbFPn9UX6dff2sedQJ/uPdI3I65cW5TM9Vj4irz1ApS+nNKakkGh5sYZt\nSs7r63Naj3axu60j8dXJroMd7Dl0/OO9h0em7AfMrRnLz2++dET/jpRK38wWAv8C5APfd/dvDHr+\nJuBmoBc4DNzo7hvMrBZ4EdiY2HSlu9+UnugyGuqiEZo0bFOylLtzuLPnuCLffaiDPYMe7znUQXfv\nifeZipYXMWlMCWdVlvDmmrFMGlPMpDEljCsrIs/Sn7eytDD9LzrIkKVvZvnAPcDlQDOwyswa3H1D\n0mY/dfd7E9svAu4GFiae2+zuc9MbW0ZLPBrhNy/tDjqGhJi7c6Srl7b2bg62d7/+Z0fPoOX+x23t\nPceWDxztpr2794TXrCgpYNKYEiaNKebiuvH9xZ5Ynph4HKsopjA/9y57pnKkfxGwyd2bAMzsIWAx\ncKz03b0tafsIoFsz5oi6WISHG7s42N49KkchEg7dvX3sPNDB9v1H2d56lOb97ew70tVf2h0nlntv\n3xtXSkVJAWNKCqksLWRMaQHTJpRRWdq/PDFxdD6xov+IfWJFMZHi8J7ZTuWdVwHbk5abgYsHb2Rm\nNwNfAIqAdyc9FTezNUAb8FV3f/r048poOzZf7t4jzK0ZG3AayRZ9fc6eQ53HSn17a/txBb/zYDvJ\nPZ6fZ4wrK6KytIAxpYWMjxRROyFyrMQrSwuTSr3wuOXykgLyR+JcS45K2687d78HuMfMPgp8FbgO\n2AlMdfd9ZjYf+LmZzR70LwPM7EbgRoCpU6emK5KkQV1iJMGWvYdV+nKMu3PgaHeiyF8v9O3722lu\nPUrzgfYTPtQ3aUwxNePKuCg+nupxpdSMK6N6fP+fkytLKMjBUymZKJXS3wHUJC1XJ9adykPAdwDc\nvRPoTDxebWabgTcBjck7uPt9wH0A9fX1OjWUQaaOLyM/zzRWP4f19PbR1tFz3Lnxg4POjQ+cbmnr\n6GFPWwfN+9s53Nlz3OuMLSukZlwZ50yu4PJZk6geX0bNuFJqxpdRNbaUksL8gN6hJEul9FcBM80s\nTn/ZXwt8NHkDM5vp7q8kFq8EXkmsjwGt7t5rZnXATKApXeFl5BUV5FEzrlRj9bPMnkMd/GHTXvYf\nOXmJtyWdMz/SdeKFzmQFeXbstMqY0kKqxpZySd2E/qP18WXHjtjHlOiaTzYYsvTdvcfMlgKP0z9k\n8353X29mdwCN7t4ALDWz9wLdwH76T+0AvAO4w8y6gT7gJndvHYk3IiNH8+VmB3dnzfYDPPDMVlb8\needxQxArivvPlVeU9J8frxlfNug8edJ587L+PwfWlRbmY6Zz5rkipXP67r4CWDFo3deSHn/+FPst\nB5afSUAJXl2snJVNrfT1OXm6YJZxOnt6+eW6nTzwx60833yQ8uIC/uriaVxTX8OUsSWUFxfofLkc\nE95xS5KyePT1+XKnjC0NOo4k7DzYzk9WbuPBZ7ex70gXMyaWc+fi2fzFvGrKQzwkUd6YfjJkSHWx\n14dtqvSD5e48u6WVB/64lcfX76bPnfeeO4nr3lLLpTMm6DSMDEmlL0M6e1IFBXlGw9rXuHRGNOg4\nodTe1cuja3fww2e28tKuQ1SWFvLJt8X560umUTO+LOh4kkVU+jKkCeXFfOJtcb73dBMfuXiqxuuP\nou2tR/nxyld5eNV2DrZ3c85ZFXzjQ+ezeG4VpUUaAinDp9KXlHz23TP4+Zod3PboC/znZy7VBd0R\n5O78YdM+fvjMVn7z0m7yzFg4+yyue2stF9aO0ykcOSMqfUlJRUkht15xDn/78DqWrW7mmgtrht5J\nhuVwZw8/e66ZB57ZyuaWI0yIFHHzZTP4q0umMrlS11IkPVT6krIPzq3ip3/axl2PvcSC2WdRWaYP\n46TDlr1HeOCZrSxf3cyhzh7eXF3J3de8mSvOn6xPsUraqfQlZWbG7Ytmc9W3f88/Pfkyty+aHXSk\nrPfr9bv49E+eI8/gyvMnc91ba7lg6rigY0kOU+nLsMyeUslfXTyNH/1xKx++sIZzJ48JOlLWWrNt\nP597aA3nTRnD966rZ2JFSdCRJAT0MT0Zti++701UlhZyW8N63HV/vNOxde8RbnigkYkVJfzg+gtV\n+DJqVPoybGPLivjywnN4dksrDeteCzpO1tl3uJPr/u1Z3J0HPnER0fLioCNJiKj05bRcU1/DnOpK\n/s+KFzky6Ba7cmrtXb184oFGdh3s4AfXX3hskhqR0aLSl9OSn2d8fdFsdrd18u3/3hR0nKzQ09vH\nZx98jj83H+DbH7mAebpgKwFQ6ctpu2DqOP5yfjU/+H0Tm1sOBx0no7k7t/9iPU++uIfbF83mfbPP\nCjqShJRKX87IlxeeQ0lhPrfrou4b+s5vN/PvK7fxqXfW8bG31AYdR0JMpS9nJFZRzN++9008/cpe\nfr1hd9BxMtLP1+zgHx/byKI3T+ErC84JOo6EnEpfztjH3jKNsydVcOcvN9DR/cZT74XNM5v28qVl\n67ikbjzf/Ms5umeRBE6lL2esID+P2xfNpnl/O/f+dnPQcTLGS7va+NSPVxOPRvju39RTXKBbKkjw\nVPqSFm+ZPoGr3jyF7zy1me2tR4OOE7idB9u5/v5VlBXn88OPX0Rlqe5TJJlBpS9p83dXnEOeGXf+\nckPQUQLV1tHN9fev4nBnDz/8+EWabUwyikpf0mZyZSmffc8Mfr1hN799uSXoOIHo6unjUz9azeaW\nw3z3b+br3kSScVT6klY3vC1OPBrh6w3r6erpCzrOqHJ3vrxsHX9s2sc/Xj1HU0tKRlLpS1oVF+Rz\n21WzaNp7hPv/sCXoOKPqm49v5OdrX+NLC87mQ/Oqg44jclIqfUm7y86eyHvPncS3fvMKuw52BB1n\nVPx45av861Ob+ejFU/nMZdODjiNySimVvpktNLONZrbJzG45yfM3mdmfzWytmf3ezGYlPXdrYr+N\nZrYgneElc33tA7Po6XP+73+9GHSUEffEht3c9ugLvOecidyxaLbmsJWMNmTpm1k+cA/wfmAW8JHk\nUk/4qbuf7+5zgX8E7k7sOwu4FpgNLAT+NfF6kuOmTijjpndO59G1r/Gnpn1Bxxkxa7bt57MPPsf5\nVZV8+6MXUJCvfzxLZkvlJ/QiYJO7N7l7F/AQsDh5A3dvS1qMAAM3YVkMPOTune6+BdiUeD0JgU+/\nczpVY0u5rWE9Pb25d1F38EQoZUWaiE4yXyqlXwVsT1puTqw7jpndbGab6T/S/9xw9pXcVFqUz99/\n4Fxe2nWIf1/5atBx0koToUi2Stu/Rd39HnefDnwF+Opw9jWzG82s0cwaW1rCOb47Vy2YfRZvnxnl\n7ideZu/hzqDjpEV7Vy83JCZC+f51mghFsksqpb8DqElark6sO5WHgA8OZ193v8/d6929PhaLpRBJ\nsoWZcdtVszna1cs3H9sYdJwz1tvnfPbBNaxrPsC3PnIB86dpIhTJLqmU/ipgppnFzayI/guzDckb\nmNnMpMUrgVcSjxuAa82s2MziwEzg2TOPLdlkxsRyPvG2OA83bmft9gNBxzlt7s5tDS/w5Iu7+fqi\n2SzQRCiShYYsfXfvAZYCjwMvAo+4+3ozu8PMFiU2W2pm681sLfAF4LrEvuuBR4ANwGPAze6ue++G\n0GffPYOJFcXc9ugL9PVl32Qr7s63/3uTJkKRrGeZNttRfX29NzY2Bh1DRsDP1+zgfz68lruWnM+H\nL5wadJyU7W7r4CvLn+epjS18cO4U7r5mru6LLxnHzFa7e/1Q22lQsYyaxXOncGHtOO56bCMHj3YH\nHWdI7s6ja3fwvn/6HSub9vH1RbNV+JL1VPoyasyM2xfN5sDRLu5+IrMv6rYe6WLpT9fw+YfWUheL\nsOJzb+e6t9aq8CXr6dMkMqpmT6nkry+Zxo9XvsqHL5zKrCmZd+vhJzfs5paf/ZmD7V18eeHZfOod\n08lX2UuO0JG+jLovXP4mxpYVcXvDejLpmtKhjm6+9B/r+OSPGolVFNOw9G185rIZKnzJKSp9GXVj\ny4r40oKzeXZrKw3rXgs6DtA/gfnCf36a5c81s/RdM3j05ks1AYrkJJ3ekUBcU1/Dg89u485fvkhn\ndx9XzJlMefHo/zi2d/Vy12Mv8cNntlIXjbD802/lgqn6wJXkLg3ZlMBseK2NpQ8+R1PLEUoK83j/\neZO5en41b6mbMCoXTJ/btp8vPrKOLXuP8PFLa/nygnMoLdJNYCU7pTpkU6UvgXJ31mw/wLLVzfxi\n3Wsc6uhhSmUJH5pXzZL51SNyX5vOnl7+5clXuPe3m5lcWco3/3IOb52uqQ0lu6n0Jet0dPfyxIbd\nLFvdzNOvtNDnMH/aOK6eX82VcyYzpqTwjP+ODa+18YVH1vLSrkN8uL6Gr37gXCrS8LoiQVPpS1bb\n3dbBf67ZwbLVzWzac5jigjwWzD6LJfOreduM6LBH1PT09vHd3zXxz0++zNiyIr7xofN5z7mTRii9\nyOhT6UtOcHeebz7IstXNNKx7jYPt3Zw1poS/mFfFknnVzJhYPuRrNLUc5ov/sY412w5w5ZzJ/O/F\n5zEuUjQK6UVGj0pfck5nTy+/eXEPy1Y389uXW+jtc+bWjOXq+dVcNWcKlWXHn6bp63N+9MetfOOx\nlygpzOfOxedx1ZunBBNeZISp9CWn7TnUwaNrXmPZ6mY27j5EUUEel8+axNXzqnn7zCi72jr48rLn\neWbzPt51doy7lsxh4piSoGOLjBiVvoSCu7P+tTaWrW7m0bU72H+0m1hFMe1dvbg7f/+BWXz4whrM\n9KlayW2plr4+nCVZzcw4r6qS86oq+bsrzuW/X9rD8ueayTP46pWzqBlfFnREkYyi0pecUVSQx8Lz\nzmLheZrRSuRUdO8dEZEQUemLiISISl9EJERU+iIiIaLSFxEJEZW+iEiIqPRFREJEpS8iEiIZdxsG\nM2sBXg06xylEgb1BhzhNyh6MbM2erbkhvNmnuXtsqI0yrvQzmZk1pnJvi0yk7MHI1uzZmhuUfSg6\nvSMiEiIqfRGREFHpD899QQc4A8oejGzNnq25QdnfkM7pi4iEiI70RURCRKUvIhIiKn0RkRBR6aeJ\nmc0ys0fM7DtmdnXQeYbDzN5uZvea2ffN7Jmg86TKzC4zs6cT2S8LOs9wmNm5idzLzOzTQecZDjOr\nM7MfmNmyoLOkItvyJhuJnxOVPmBm95vZHjN7YdD6hWa20cw2mdktQ7zM+4Fvu/ungY+NWNhB0pHd\n3Z9295uAXwIPjGTepHzp+J47cBgoAZpHKutgafqev5j4nl8DXDqSeZOlKXuTu98wsknf2HDeRybk\nTTbM7On/OXH30H8B7wDmAS8krcsHNgN1QBGwDpgFnE9/OSZ/TUx83QN8E/hDNmVP2u8RoCJbcgN5\nif0mAT/Jtu85sAj4L+Cj2ZY9sd+y0cp9Ju8jE/KeSfZ0/5wE/g3IlC+gdtB/hLcAjyct3wrcmsLr\n5AOPZlt2YCrwvWzLndiuaLT/h05X9sS2v8rG7EGX6HDfR9B5z/S/Qbp+TnR659SqgO1Jy82JdSdl\nZrVmdh/wI/qP9oM0rOwJNwD/NmKJUjPc7/mHzOy7wI+B/zfC2YYy3OyXmdm3EvlXjHS4IQw3+wQz\nuxe4wMxuHelww3DS95HBeZOdKnvaf04K0vEiAu6+Fbgx6Byny91vCzrDcLn7z4CfBZ3jdLj7U8BT\nAcc4Le6+D7gp6Bypyra8yUbi50RH+qe2A6hJWq5OrMsG2Zo9W3ODsmeCbH4fo5ZdpX9qq4CZZhY3\nsyLgWqAh4Eypytbs2ZoblD0TZPP7GL3sQV/QyIQv4EFgJ9BN/7m0GxLrrwBepv+q+v8KOmcuZc/W\n3MqeGV/Z/D6Czq4bromIhIhO74iIhIhKX0QkRFT6IiIhotIXEQkRlb6ISIio9EVEQkSlLyISIip9\nEZEQUemLiITI/wcpy7c7xXwFrgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c24c622b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(C_list, errors)\n",
    "plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "optimal C : 0.000037\n",
      "LR took 7.973318 seconds.\n"
     ]
    }
   ],
   "source": [
    "i_opt = np.argmin(errors)\n",
    "C_opt = C_list[i_opt] #0.000018#\n",
    "print('optimal C : %f'%C_opt)\n",
    "model_cnn = sklearn.linear_model.LogisticRegression(penalty='l2', C=C_opt, class_weight=class_weight)\n",
    "t = time.time()\n",
    "model_cnn.fit(cnn_train, train_labels)\n",
    "print(\"LR took %f seconds.\"%(time.time()-t))\n",
    "predictions = model_cnn.predict(cnn_test)\n",
    "err = (test_labels!=predictions).sum()/len(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.245"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn_proba_val = model_cnn.predict_proba(cnn_val)[:,:-1]\n",
    "cnn_proba_test = model_cnn.predict_proba(cnn_test)[:,:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l = 6\n",
    "color = False\n",
    "train_reduc, val_reduc, test_reduc = split.load_images(train_ids, val_ids, test_ids, l, color)\n",
    "train_reduc = train_reduc.reshape((train_reduc.shape[0], l*l))\n",
    "val_reduc = val_reduc.reshape((val_reduc.shape[0], l*l))\n",
    "test_reduc = test_reduc.reshape((test_reduc.shape[0], l*l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49770546516478931"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lr = sklearn.linear_model.LogisticRegression(penalty='l2', C=1e10, class_weight=class_weight)\n",
    "model_lr.fit(train_reduc, train_labels)\n",
    "predictions = model_lr.predict(val_reduc)\n",
    "error = (val_labels!=predictions).sum()/len(val_labels)\n",
    "error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reduc_proba_val = model_lr.predict_proba(val_reduc)[:,:-1]\n",
    "reduc_proba_test = model_lr.predict_proba(test_reduc)[:,:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_mix = np.concatenate([cnn_proba_val, reduc_proba_val], axis=1)\n",
    "test_mix = np.concatenate([cnn_proba_test, reduc_proba_test], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.245"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_ensemble = sklearn.linear_model.LogisticRegression(penalty='l2', C=1e10, class_weight=class_weight)\n",
    "model_ensemble.fit(val_mix, val_labels)\n",
    "predictions = model_ensemble.predict(test_mix)\n",
    "error = (test_labels!=predictions).sum()/len(test_labels)\n",
    "error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probas = model_ensemble.predict_proba(test_mix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[925,  32,  11,  72],\n",
       "       [ 46, 416,  15,  54],\n",
       "       [ 27,  17, 194,  24],\n",
       "       [163, 106,  21, 277]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.confusion_matrix(test_labels, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "confidence = np.max(probas, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1.,   0.,   0.,   0.,   0.,   1.,   0.,   0.,   0.,   0.,   1.,\n",
       "          1.,   4.,   5.,   5.,   6.,   2.,   4.,   6.,   3.,   0.,   8.,\n",
       "          6.,   6.,  12.,   6.,   8.,  12.,  19.,  17.,  17.,  19.,  15.,\n",
       "         18.,  17.,  19.,  18.,  18.,  18.,  20.,  12.,  15.,  14.,  17.,\n",
       "         24.,  29.,  16.,  21.,  18.,  19.,  32.,  19.,  15.,  18.,  19.,\n",
       "         25.,  22.,  12.,  22.,  24.,  27.,  22.,  27.,  20.,  31.,  31.,\n",
       "         27.,  22.,  31.,  35.,  35.,  38.,  32.,  28.,  38.,  36.,  40.,\n",
       "         27.,  38.,  44.,  40.,  48.,  43.,  44.,  52.,  62.,  64.,  61.,\n",
       "         63.,  74.,  75.,  75.,  76.,  67.,  57.,  36.,  45.,  44.,  24.,\n",
       "         16.]),\n",
       " array([ 0.28020742,  0.28668255,  0.29315769,  0.29963282,  0.30610796,\n",
       "         0.3125831 ,  0.31905823,  0.32553337,  0.33200851,  0.33848364,\n",
       "         0.34495878,  0.35143392,  0.35790905,  0.36438419,  0.37085933,\n",
       "         0.37733446,  0.3838096 ,  0.39028474,  0.39675987,  0.40323501,\n",
       "         0.40971015,  0.41618528,  0.42266042,  0.42913555,  0.43561069,\n",
       "         0.44208583,  0.44856096,  0.4550361 ,  0.46151124,  0.46798637,\n",
       "         0.47446151,  0.48093665,  0.48741178,  0.49388692,  0.50036206,\n",
       "         0.50683719,  0.51331233,  0.51978747,  0.5262626 ,  0.53273774,\n",
       "         0.53921288,  0.54568801,  0.55216315,  0.55863828,  0.56511342,\n",
       "         0.57158856,  0.57806369,  0.58453883,  0.59101397,  0.5974891 ,\n",
       "         0.60396424,  0.61043938,  0.61691451,  0.62338965,  0.62986479,\n",
       "         0.63633992,  0.64281506,  0.6492902 ,  0.65576533,  0.66224047,\n",
       "         0.66871561,  0.67519074,  0.68166588,  0.68814101,  0.69461615,\n",
       "         0.70109129,  0.70756642,  0.71404156,  0.7205167 ,  0.72699183,\n",
       "         0.73346697,  0.73994211,  0.74641724,  0.75289238,  0.75936752,\n",
       "         0.76584265,  0.77231779,  0.77879293,  0.78526806,  0.7917432 ,\n",
       "         0.79821833,  0.80469347,  0.81116861,  0.81764374,  0.82411888,\n",
       "         0.83059402,  0.83706915,  0.84354429,  0.85001943,  0.85649456,\n",
       "         0.8629697 ,  0.86944484,  0.87591997,  0.88239511,  0.88887025,\n",
       "         0.89534538,  0.90182052,  0.90829566,  0.91477079,  0.92124593,\n",
       "         0.92772106]),\n",
       " <a list of 100 Patch objects>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEGhJREFUeJzt3X2MZXddx/H3h5aK1kqfhs2m22WrFrAabWHSSDCClJJK\ntV2gadqo2ZrqRqMIAWNX8A9FjYt/gCQSzNoiGwKUUiFdEdDN0oZAoLKlW6At0Ae3YUvbXZCGByNY\n/PrHnMKwzPSeO3PvnLm/fb+SzT333HN7P3sy/cxvf+fhpqqQJM2+Jw0dQJI0GRa6JDXCQpekRljo\nktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqRHHr+WHnX766bVly5a1/EhJmnm33XbbV6pqbtR2a1ro\nW7ZsYf/+/Wv5kZI085I80Gc7p1wkqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQ\nJakRa3qlqCStR1t2/Ov3lg/uvHjAJKvjCF2SGuEIXdIxY/FIvEWO0CWpERa6JDXCKRdJTWt9mmUx\nR+iS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpESMLPckzkxxY9OfrSV6V5NQke5Pc0z2eshaBJUlL\nG1noVfWFqjq3qs4FngP8N/B+YAewr6rOBvZ1zyVJAxl3yuUC4L6qegC4FNjdrd8NbJ1kMEnSeMYt\n9CuAd3fLG6rqoW75YWDDxFJJksbWu9CTnABcArz36NeqqoBa5n3bk+xPsv/IkSMrDipJemLjjNB/\nFfh0VT3SPX8kyUaA7vHwUm+qql1VNV9V83Nzc6tLK0la1jiFfiXfn24B2ANs65a3ATdNKpQkaXy9\nCj3JicCFwPsWrd4JXJjkHuBF3XNJ0kB63T63qr4FnHbUuq+ycNaLJGkd8EpRSWqEhS5JjbDQJakR\nfgWdJC2y+CvrDu68eMAk43OELkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIzwPXdLMmuVz\nxqfBEbokNcJCl6RGOOUiqTmLp2KOJY7QJakRFrokNcIpF0kz5VidTunDEbokNaLvl0SfnOTGJJ9P\ncneS5yY5NcneJPd0j6dMO6wkaXl9p1zeDHy4qi5LcgLwY8BrgX1VtTPJDmAHcM2UckrSE3IqpscI\nPclTgV8GrgOoqu9U1aPApcDubrPdwNZphZQkjdZnyuUs4AjwT0luT3JtkhOBDVX1ULfNw8CGpd6c\nZHuS/Un2HzlyZDKpJUk/pE+hHw88G3hrVZ0HfIuF6ZXvqaoCaqk3V9Wuqpqvqvm5ubnV5pUkLaNP\noR8CDlXVrd3zG1ko+EeSbAToHg9PJ6IkqY+RhV5VDwNfSvLMbtUFwF3AHmBbt24bcNNUEkqSeul7\nlssrgHd2Z7jcD/w2C78MbkhyNfAAcPl0IkqS+uhV6FV1AJhf4qULJhtHkrRSXikqSY2w0CWpEd6c\nS5KWMWtfcecIXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJ\naoSFLkmNsNAlqRHebVHSurf4rodaniN0SWpErxF6koPAN4DvAo9V1XySU4H3AFuAg8DlVfW16cSU\nJI0yzgj9V6rq3Kp6/LtFdwD7qupsYF/3XJI0kNVMuVwK7O6WdwNbVx9HkrRSfQu9gH9PcluS7d26\nDVX1ULf8MLBh4ukkSb31Pcvll6rqwSRPA/Ym+fziF6uqktRSb+x+AWwH2Lx586rCSpKW12uEXlUP\ndo+HgfcD5wOPJNkI0D0eXua9u6pqvqrm5+bmJpNakvRDRhZ6khOTnPT4MvBi4HPAHmBbt9k24KZp\nhZQkjdZnymUD8P4kj2//rqr6cJJPATckuRp4ALh8ejElSaOMLPSquh/4hSXWfxW4YBqhJEnj89J/\nSYNafFn/wZ0XD5hk9nnpvyQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjfA8dEka03o9d94R\nuiQ1wkKXpEY45SJp6hZPUUC/aYqj37NerafpF0foktQIC12SGuGUi6R1Y1amWdYrR+iS1AgLXZIa\nYaFLUiN6F3qS45LcnuQD3fOzktya5N4k70lywvRiSpJGGWeE/krg7kXP3wC8qap+GvgacPUkg0mS\nxtOr0JNsAi4Gru2eB3ghcGO3yW5g6zQCSpL66TtC/zvgT4D/656fBjxaVY91zw8BZ0w4myRpDCPP\nQ0/ya8DhqrotyQvG/YAk24HtAJs3bx47oKT1bSWXvnu++XT0GaE/D7gkyUHgehamWt4MnJzk8V8I\nm4AHl3pzVe2qqvmqmp+bm5tAZEnSUkYWelX9aVVtqqotwBXAR6rqN4Cbgcu6zbYBN00tpSRppNWc\nh34N8Ook97Iwp37dZCJJklZirHu5VNUtwC3d8v3A+ZOPJElaCW/OJanXgc1xD3564HPteem/JDXC\nQpekRljoktQIC12SGmGhS1IjPMtF0tiOxTNYZuHv7AhdkhrhCF06hszCKFMr5whdkhphoUtSIyx0\nSWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgvLJJY2TfXt8p9MbtGjtCTPCXJfyS5I8mdSf6iW39W\nkluT3JvkPUlOmH5cSdJy+ozQvw28sKq+meTJwMeSfAh4NfCmqro+yT8AVwNvnWJWSSx/+X6fr45T\n20aO0GvBN7unT+7+FPBC4MZu/W5g61QSSpJ66XVQNMlxSQ4Ah4G9wH3Ao1X1WLfJIeCM6USUJPXR\nq9Cr6rtVdS6wCTgfeFbfD0iyPcn+JPuPHDmywpiSpFHGOm2xqh4FbgaeC5yc5PE5+E3Ag8u8Z1dV\nzVfV/Nzc3KrCSpKW1+csl7kkJ3fLPwpcCNzNQrFf1m22DbhpWiElSaP1OctlI7A7yXEs/AK4oao+\nkOQu4PokfwXcDlw3xZzSSJ4/rWPdyEKvqs8A5y2x/n4W5tMlSeuAl/5LUiO89F/SsrwoabY4Qpek\nRjhClwa23MHccQ/yroeDwo7oh+UIXZIaYaFLUiMsdElqhIUuSY2w0CWpEZ7lIq3Seji7RAJH6JLU\nDEfommmzet7zrObWExv6X2uO0CWpERa6JDXCQpekRljoktQIC12SGuFZLtIEHX32ynJ3T+z7/knl\n0LHBEbokNWJkoSc5M8nNSe5KcmeSV3brT02yN8k93eMp048rSVpOnymXx4DXVNWnk5wE3JZkL3AV\nsK+qdibZAewArpleVGllhr7YQ1orI0foVfVQVX26W/4GcDdwBnApsLvbbDewdVohJUmjjXVQNMkW\n4DzgVmBDVT3UvfQwsGGZ92wHtgNs3rx5pTnVMEfQo3mQU330Piia5MeBfwZeVVVfX/xaVRVQS72v\nqnZV1XxVzc/Nza0qrCRpeb0KPcmTWSjzd1bV+7rVjyTZ2L2+ETg8nYiSpD5GTrkkCXAdcHdVvXHR\nS3uAbcDO7vGmqSTU1PSZ6ljJdMisTKEsN40x7n6R1os+c+jPA34L+GySA92617JQ5DckuRp4ALh8\nOhElSX2MLPSq+hiQZV6+YLJxJEkr5aX/WpEnusT9ibYb2mryrLe/i3Q0L/2XpEY4Qhew+tHnWo5e\nHSlLS3OELkmNsNAlqRFOuahJTstoaENcj+EIXZIaYaFLUiOcclFvaz2NsR5uIeDUjWaJI3RJaoSF\nLkmNcMpFM8GpD2k0R+iS1AgLXZIaYaFLUiMsdElqhAdFG9LCV6et93zjau3vo/XNEbokNWJkoSd5\nW5LDST63aN2pSfYmuad7PGW6MSVJo/QZob8duOiodTuAfVV1NrCvey5JGtDIQq+qjwL/ddTqS4Hd\n3fJuYOuEc0mSxrTSOfQNVfVQt/wwsGFCeSRJK7Tqg6JVVUAt93qS7Un2J9l/5MiR1X6cJGkZKy30\nR5JsBOgeDy+3YVXtqqr5qpqfm5tb4cdJkkZZaaHvAbZ1y9uAmyYTR5K0Un1OW3w38AngmUkOJbka\n2AlcmOQe4EXdc0nSgEZeKVpVVy7z0gUTziJJWgUv/Z8Rq/k6Ni8/l44NXvovSY2w0CWpERa6JDXC\nQpekRnhQdMrGvUd5nwOefQ5yeiBUOvY4QpekRljoktQIp1wmxCkOSUNzhC5JjbDQJakRTrmswmqm\nWZyikTRpjtAlqREWuiQ1wimXdWY1d1WUdGxzhC5JjXCEvoY8ECppmhyhS1IjVlXoSS5K8oUk9ybZ\nMalQkqTxrXjKJclxwFuAC4FDwKeS7KmquyYVbrFJfQWbX98mqVWrGaGfD9xbVfdX1XeA64FLJxNL\nkjSu1RT6GcCXFj0/1K2TJA1g6me5JNkObO+efjPJF1b933zDsi+dDnxlhe8d2g9lX8dZjzZyv69j\nZh/GMZV9Av8vP73PRqsp9AeBMxc939St+wFVtQvYtYrP6S3J/qqaX4vPmjSzD8PswzD7dKxmyuVT\nwNlJzkpyAnAFsGcysSRJ41rxCL2qHkvyh8C/AccBb6uqOyeWTJI0llXNoVfVB4EPTijLJKzJ1M6U\nmH0YZh+G2acgVTV0BknSBHjpvyQ1YiYLfdQtB5L8XpLPJjmQ5GNJzhki51L63i4hycuTVJJ1czS9\nx36/KsmRbr8fSPI7Q+RcSp/9nuTyJHcluTPJu9Y643J67Pc3LdrnX0zy6BA5l9Ij++YkNye5Pcln\nkrxkiJxL6ZH96Un2dblvSbJpiJw/oKpm6g8LB2DvA34SOAG4AzjnqG1+YtHyJcCHh87dN3u33UnA\nR4FPAvND5x5jv18F/P3QWVeY/WzgduCU7vnThs49zs/Mou1fwcIJCjORnYX56N/vls8BDg6de4zs\n7wW2dcsvBN4xdO5ZHKGPvOVAVX190dMTgfVyoKDv7RL+EngD8D9rGW6EWb7VQ5/svwu8paq+BlBV\nh9c443LG3e9XAu9ek2Sj9clewE90y08FvryG+Z5In+znAB/plm9e4vU1N4uF3uuWA0n+IMl9wN8C\nf7RG2UYZmT3Js4Ezq2q93RWs760eXt79E/TGJGcu8foQ+mR/BvCMJB9P8skkF61ZuifW+xYbSZ4O\nnMX3S2ZofbL/OfCbSQ6xcMbcK9Ym2kh9st8BvKxbfilwUpLT1iDbsmax0HupqrdU1U8B1wB/NnSe\nPpI8CXgj8Jqhs6zQvwBbqurngb3A7oHzjON4FqZdXsDCKPcfk5w8aKLxXQHcWFXfHTrIGK4E3l5V\nm4CXAO/o/j+YBX8MPD/J7cDzWbhSftB9Pys7brFetxxY5Hpg61QT9Tcq+0nAzwG3JDkI/CKwZ50c\nGB2536vqq1X17e7ptcBz1ijbKH1+Zg4Be6rqf6vqP4EvslDwQxvn5/0K1s90C/TLfjVwA0BVfQJ4\nCgv3Shlan5/3L1fVy6rqPOB13bphD0gPPYm/goMVxwP3s/BPy8cPVvzsUducvWj514H9Q+fum/2o\n7W9h/RwU7bPfNy5afinwyaFzj5H9ImB3t3w6C//cPm0WsnfbPQs4SHdtyXr403O/fwi4qlv+GRbm\n0Af/O/TMfjrwpG75r4HXD5576AAr3NkvYWEEdR/wum7d64FLuuU3A3cCB1g4WLFsaa637Edtu24K\nved+/5tuv9/R7fdnDZ15jOxhYbrrLuCzwBVDZx7nZ4aFueidQ2ddwX4/B/h49zNzAHjx0JnHyH4Z\ncE+3zbXAjwyd2StFJakRsziHLklagoUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1Ij/h+v\nIzPxSfrW6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c23c46240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(confidence, bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/salon/.virtualenvs/solar/lib/python3.5/site-packages/ipykernel_launcher.py:5: RuntimeWarning: invalid value encountered in true_divide\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "N = len(confidence)\n",
    "confidence_order = np.argsort(-confidence)\n",
    "sorted_error = (predictions[confidence_order]!=test_labels[confidence_order])\n",
    "nb_errors = np.cumsum(sorted_error)\n",
    "error_rate = nb_errors/np.arange(N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f1c20036080>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeYVOXZx/HvvQssZWHpRWApCgJKdcGOBUFiQ40FJbHE\niCUmrzGa10gw1thiYnk1iorGhoqRiBHFAoqFLijSe5NelrbLtvv9Yw7D7LKwA+zMbPl9rmsv5rTZ\n+ywwv33Oc87zmLsjIiJyIEmJLkBERMo+hYWIiJRIYSEiIiVSWIiISIkUFiIiUiKFhYiIlEhhISIi\nJVJYiIhIiRQWIiJSoiqJLqC0NGzY0Fu3bp3oMkREypXp06dvdPdGJe1XYcKidevWTJs2LdFliIiU\nK2a2PJr9dBlKRERKFNOwMLP+ZjbfzBaZ2Z3FbL/NzOaY2Q9m9rmZtYrYlm9mM4Ov0bGsU0REDixm\nl6HMLBl4BugLrAKmmtlod58TsdsMIMPdd5nZTcCjwOXBtix37xar+kREJHqxbFn0Aha5+xJ3zwHe\nAgZE7uDu4919V7A4CWgRw3pEROQQxTIsmgMrI5ZXBev25zrgo4jl6mY2zcwmmdmFxR1gZoODfaZt\n2LDh8CsWEZFilYm7oczsF0AGcFrE6lbuvtrM2gLjzGyWuy+OPM7dhwHDADIyMjSLk4hIjMSyZbEa\naBmx3CJYV4iZnQUMAS5w99171rv76uDPJcAXQPcY1ioiIgcQy7CYCrQzszZmVg0YCBS6q8nMugPP\nEwqK9RHr65lZSvC6IXAyENkxLiIiwAff/8SkJZti/n1idhnK3fPM7BZgLJAMDHf32WZ2HzDN3UcD\njwGpwEgzA1jh7hcAHYHnzayAUKA9XOQuKhGRSu+uUbN4c/IKalRNZva9Z5OUZDH7XjHts3D3McCY\nIuvujnh91n6O+xboHMvaRETKs5Wbd/Hm5BUAvHvTiTENCigjHdwiIhK9z+as49evhoY3mnDHGaQ3\nqBnz76mwEBEpJ7Jy8nniswU8P2EJAL86uU1cggIUFiIi5cKkJZsYOGxSePnxS7vy8+Pi9xyzwkJE\npIy77Z2ZvPdd6MmDdo1TefrK7nRoWieuNSgsRETKKHfn5je+46Mf1wLw53M78utT2yakFoWFiEgZ\ntGj9ds76+4Tw8oyhfalXq1rC6tF8FiIiZcxPW7MKBcX4209PaFCAWhYiImVKTl4Blz0/EYD+xzTl\nySu6kVIlOcFVKSxERMoEd+eL+Ru49pWpAJzXpRlPX9GdYHSLhFNYiIgk2PNfLuahj+aFlxvVTuGJ\ny7uVmaAAhYWISEJ9MnttoaB4+OLODOyVnsCKiqewEBFJkMysXAa/Nh2AD245hc4t0hJc0f4pLERE\n4iw7N5/XJy3ngQ/nAvDQxZ3LdFCAwkJEJK7y8gvoMPTj8HK/Tk24ogxedipKYSEiEkf3fDA7/HrU\nzSfRPb1eAquJnsJCRCROftqaxeuTVpBSJYl59/cvU3c7lURPcIuIxMH27FxOengcAO/ccGK5CgpQ\nWIiIxMWlz4Weys5oVY+uLesmuJqDp7AQEYmxhz6ay7y122nXOJWRN56Y6HIOifosRERiwN15fdJy\nhr6/t0P73zefVO4uP+2hloWISAy8OWVFoaB47+aTqFO9agIrOjxqWYiIlLJvF29kyKgfgVBndq82\n9RNc0eFTWIiIlJLc/ALufn82I6asAOCZK3tUiKAAhYWIyGHbtGM3n89bzx/f/SG87tFLunBO56YJ\nrKp0KSxERA7D058v5PFPFxRa982dZ9K8bo0EVRQbCgsRkUP01zFzGTZhSXj56Su6c37XIxJYUewo\nLEREDsHHP64JB8Wnv+9Nuya1E1xRbCksREQO0uXPT2Ty0s0AfFIJggL0nIWIyEH5dM66cFA8ObAb\n7StBUIBaFiIiUVuwbjvXvzoNgO+G9qV+rWoJrih+1LIQEYmCu/O7ETMAGHJOx0oVFBDjsDCz/mY2\n38wWmdmdxWy/zczmmNkPZva5mbWK2Ha1mS0Mvq6OZZ0iIgfy4ldLaPOnMcxbu50rerXk+t5tE11S\n3MXsMpSZJQPPAH2BVcBUMxvt7nMidpsBZLj7LjO7CXgUuNzM6gN/ATIAB6YHx26JVb0iIpGyc/ML\nTX+6x5BzOyWgmsSLZcuiF7DI3Ze4ew7wFjAgcgd3H+/uu4LFSUCL4PXZwKfuvjkIiE+B/jGsVUQk\nzN05/+mvC6178/rjmXd/f1JTKmdXbyzPujmwMmJ5FXD8Afa/DvjoAMc2L9XqRET2Y8SUlSxcv4Nq\nyUnMf6B8TX8aK2UiIs3sF4QuOZ12kMcNBgYDpKenx6AyEalsduXkcdeoWQDM/EtfBUUglpehVgMt\nI5ZbBOsKMbOzgCHABe6++2COdfdh7p7h7hmNGjUqtcJFpHL6auEGOt09FoCHLu5MzWpl4vfpMiGW\nYTEVaGdmbcysGjAQGB25g5l1B54nFBTrIzaNBfqZWT0zqwf0C9aJiMREdm4+/xuMGnt0k9oM7Nmy\nhCMql5jFprvnmdkthD7kk4Hh7j7bzO4Dprn7aOAxIBUYGTT1Vrj7Be6+2czuJxQ4APe5++ZY1Soi\ncvpjX7B2Wza39W3P7/q0S3Q5ZU5M21juPgYYU2Td3RGvzzrAscOB4bGrTkQEtu7K4Y3JK1i7LRuA\n60+tfM9QREMX5ESk0srNL6DbfZ+Gl2fe3Zca1ZITWFHZpbAQkUrH3dm6K5erhk8Jr3v5mp7UrVm5\nhvA4GAoLEal0Xvl2Gfd+sHcwiaUPnaNbZEuggQRFpFL5+Me14aCokmRMuOMMBUUU1LIQkUrB3Rk5\nfRV/DG6Pfe26XpzaTs9nRUthISKVwkMfzQtPg/rgRccqKA6SwkJEKrzs3PxwUDxzZQ/O7dIswRWV\nP+qzEJEK7cfVmeGhxv85SEFxqBQWIlJhjZ+/nvOCocabpVXnZ50VFIdKl6FEpELauiuHa18OjRh0\n3SltuOucjgmuqHxTWIhIhVNQ4OEns4ee14nrTmmT4IrKP4WFiFQoOXkFPPLxvPCygqJ0KCxEpMK4\nY+T3jJy+Kry88MGfJbCaikVhISLl3msTlzH0/dmF1r09+ASqJusentKisBCRcu21ScsLBcUHt5xC\n5xZpCayoYlJYiEi5lJWTz7WvTGHSktC8aCOuP4ETj2yQ4KoqLrXRRKRceu7LxeGgeObKHgqKGFPL\nQkTKFXfn/Zk/8eTnC6lTvQof3dqb5nVrJLqsCk9hISLlSo/7P2XLrlwg9AyFgiI+FBYiUi7szsvn\njMe+CAfFs4N6cI6G74gbhYWIlFn5BU5eQQHPfbGEf3y2ILx+7n39NVd2nCksRKRMWrl5F6c+On6f\n9dP+fJaCIgEUFiJSpqzJzOLal6cyb+32QutHXH8CGa3r6UG7BFFYiEiZsH57Nqc8PJ6c/ILwukHH\np/PgRZ0TWJXsobAQkYSbt3Yb/Z/4KrzcvG4N3rz+eFo1qJXAqiSSwkJEEiozKzccFD3S6/LKr3pR\np3rVBFclRSksRCRh3p2+ittHfg9AlxZpvHfzyQmuSPZHPUUikhA7dudx3wehAQCb1ElhlIKiTFPL\nQkTibuOO3WQ88BkAf/pZB2447cgEVyQlUViISFzdNWoWb05eAUDdmlUZdEKrBFck0VBYiEhcuDvX\nvjKVL+ZvAKB9k1Q++f1pCa5KohXTsDCz/sCTQDLwors/XGR7b+AJoAsw0N3fjdiWD8wKFle4+wWx\nrFVEYucv7//IvyYuDy+//5uT6dqybgIrkoMVs7Aws2TgGaAvsAqYamaj3X1OxG4rgGuA24t5iyx3\n7xar+kQk9jKzcul67yeF1n39v2fQol7NBFUkhyqWLYtewCJ3XwJgZm8BA4BwWLj7smBbQXFvICLl\nx+68fKYs3cx1/5pGTl4B53RuyphZa8PbP7utN0c1rp3ACuVwxDIsmgMrI5ZXAccfxPHVzWwakAc8\n7O7/Kc3iRKT0fLNoI4NenFxo3Z6g6NupCS9clZGIsqQUleUO7lbuvtrM2gLjzGyWuy+O3MHMBgOD\nAdLT0xNRo0il9/I3S7n3g71Xl286/Uj+ePbRfL1oI+7Qu32jBFYnpSWWYbEaaBmx3CJYFxV3Xx38\nucTMvgC6A4uL7DMMGAaQkZHhh1mviByknLyCcFDcP+AYBh3fiqQkA+DUdgqJiiSWT3BPBdqZWRsz\nqwYMBEZHc6CZ1TOzlOB1Q+BkIvo6RKRsuOblKQD8rk87fnli63BQSMUTs5aFu+eZ2S3AWEK3zg53\n99lmdh8wzd1Hm1lPYBRQDzjfzO5192OAjsDzQcd3EqE+C4WFSILlFzg3vDadz+auK7T+t2celaCK\nJF5i2mfh7mOAMUXW3R3xeiqhy1NFj/sW0CD2ImWIu3PUkDF4kQu+//3tKZqQqBI4YFgEz0rMdvcO\ncapHRMqoEVNWhoPi2zvP5Ii6NRJbkMTVAX8dcPd8YL6Z6VYjkUrszckruGtUaEAFBUXlFM1lqHrA\nbDObAuzcs1LDb4hUTOPmrSO/ANo0rMmbk1eydltW+JmJv17UWUFRSUUTFkNjXoWIlAmXPTeRKcs2\nF7vtwYuO5crjdZGhsioxLNz9SzNrAvQMVk1x9/WxLUtE4mXBuu3837hFtGpQMxwU53RuyuotWSzZ\nuJOHL+5CzWrJnNGhcYIrlUQqMSzM7DLgMeALwICnzeyOyBFiRaR8yskr4KbXp7N4Q/gKM5/+vjft\nmmgMJyksmstQQ4Cee1oTZtYI+AxQWIiUYw9+OIcXvloKwBFp1RnYK53WDWspKKRY0YRFUpHLTpvQ\n3N0i5dbSjTtZtWVXOCgA3r7hRFrW17Dhsn/RhMXHZjYWGBEsX06RB+1EpOz7csEGrh4+pdC6hy/u\nzMBe6rSWkkXTwX2HmV0MnBKsGubuo2JbloiUpk9mr2Xwa9PDy+2bpHJcq/pc3rPlAY4S2SuaJ7g/\nc/czgPfiU5KIHKrs3HwAqldNZufuPLZl5/LwR/N4f+ZPANz5sw7ceNqRiSxRyqkDhoW755tZgZml\nuXtmvIoSkYO3fNNOTnvsCwD+p087nvx8YaHtd5x9tIJCDlk0fRY7gFlm9imFn+D+XcyqEpGo5Rc4\nXy5Yz69emRZeFxkUR6RV58krutOtZd1ElCcVRDRh8R66BCVSJj0zfhGPjZ0fXu6RXpd/33QS4+ev\nZ+6a7fzmDA0dLqUjmj6Lfu4+KE71iEiUnvhsAU98trcF8ZszjuSOs0MDRJ/ZoQlndmiSqNKkAoqm\nz6KVmVVz95x4FSUiB3bNy1P4Yv4GAB69pAt9OjSmQWpKgquSiiyay1BLgG/MbDSF+yz+HrOqRGS/\nNmzfHQ6Kl6/tyRlHa8wmib1owmJx8JUEaBwAkQTKL3B+88Z3ALx380n0SK+X4Iqksojmobx7Acys\nprvvin1JIlKc7dm5dL7nk/Byd93dJHEUzaizJwIvAalAupl1BW5w95tjXZxIZffqxGXc/f7sfda/\nfE1PzCz+BUmlFc1lqCeAs4HRAO7+vZn1jmlVIpXYmswsbn1rJg1TU/hw1ppC29o1TuW/vzuFlCrJ\nCapOKqtowgJ3X1nkt5j82JQjUnm5Oze8Np1P5qwrtP7Xp7Thz+d1SlBVIiHRhMVKMzsJcDOrCvwP\nMDe2ZYlUDgUFzptTVlCjajJ/GPl9eP3RTWpzy5lHUbt6FXq3a5TACkVCogmLG4EngebAauAT4Dex\nLEqkolu5eRenPjq+2G2T/tSHpmnV41yRyIFFczfURkBPcIuUkuzc/H2C4uc9WtAgtRq/69OO1JSo\nrg6LxJX+VYrE0UNj5vL8hCUApNWoyoyhfSlwp0qyJp+Usk1hIRIns1ZlhoOiVrVkZt7dFzMjCd0C\nK2WfwkIkRrJy8vl49hpGTF7JlGWbw+ufuqI7F3Q9IoGViRy8qMPCzE4A7gGqA0+4+39iVZRIeTZ1\n2WYufW5isdvO69KM87s0i3NFIodvv2FhZk3dfW3EqtuAiwADJgMKC5EitmXn7hMUN/Ruy2U9W3Jk\no9QEVSVy+A7UsnjOzL4DHnX3bGArcAlQAGyLR3Ei5c15T30NwFkdm/Di1RkJrkak9Oz3Fgx3vxCY\nAfzXzK4CbgVSgAbAhfEpT6R8yM7N56nPF7Jic2iszWG/PC7BFYmUrgPer+fuHxAaFyoNGAUscPen\n3H1DNG9uZv3NbL6ZLTKzO4vZ3tvMvjOzPDO7pMi2q81sYfB1dfSnJBIfq7dmMXPlVq4aPoUOQz/m\n758uAOCz23qTlKQ7nKRiOVCfxQXA74E84K/Aa8BQM7sZGOLuiw/0xsGUrM8AfYFVwFQzG+3ucyJ2\nWwFcA9xe5Nj6wF+ADMCB6cGxWw7u9ERKx8J12zmyUSp5Bc767dnMXbOd61+dts9+vz+rPUc11rQv\nUvEcqM/iAaAXUAMY6+69gD+YWTvgQWBgCe/dC1jk7ksAzOwtYAAQDgt3XxZsKyhy7NnAp+6+Odj+\nKdAfGBHdaYmUjtVbs5ixYgu3vDmDLi3S+GFV5j77HJFWnWcG9aC7JiKSCuxAYZEJXAzUBNbvWenu\nCyk5KCA0ltTKiOVVwPFR1lXcsc2L7mRmg4HBAOnp6VG+tUjx8vILSDJjzbZsBr0wiRrVqjB3zd57\nOdZmZodfp6ZUYdAJ6dzZv4PmlZBK4UBhcRFwBZALXBmfcg6Ouw8DhgFkZGR4gsuRcu6oIR/td9s1\nJ7XmjrOPZuS0lZx+dGNaN6wVx8pEEm+/YREMIPj0Ybz3aqBlxHKLYF20x55e5NgvDqMWkbDs3HxW\nb82ialISDVKrUbNaMu99V/ifZlqNqjx8cWdSqiZxZocm4fXXnNwm3uWKlAmxHO5jKtDOzNoQ+vAf\nSPQtlLHAX81sz0XgfsCfSr9EqUwKCpzsvHw63T12v/t8ecfpZOXm06FpnThWJlL2xSws3D3PzG4h\n9MGfDAx399lmdh8wzd1Hm1lPQrfk1gPON7N73f0Yd99sZvcTChyA+/Z0doscikXrt3PW3ycccJ/L\nMlrQqoEuL4kUx9wrxqX+jIwMnzZt31sZRTZs303PBz8LLycZzL2/PylVknF3MrNyyckroHEdTTgk\nlY+ZTXf3Eocb0KizUuGNnB66sa5z8zRG3XwSSWbhh+bMjLo1qyWyPJFyQWEhFdqyjTt59OP5AIy+\n5WTd5ipyiBQWUuG8P3M1ExZspE/Hxtz8xncAXJ7RUkEhchgUFlKhLFy3nf95ayYA//5uFQBdW9bl\n4Z93TmRZIuWewkIqlOHfLAVCD9Ft2pnDxu27efP649WqEDlMCgupMBat38GIKaHO7HsuOCbB1YhU\nLAoLKfe27Mxh6Ps/8t8f1gDwh77tE1yRSMWjsJByzd0596mv+ClikL/f9mmXwIpEKiaFhZQrP67O\nZPXWLPp1aoKZcfLD48JB8cjPO3PMEWkJrlCkYlJYFLFg3XbS69eketXkRJdSaeXkFVA12diWlceN\nr09nxeZdDD2vEyce2YDzng7Ncd2zdT06NqsTDor3bj6JHppPQiRmFBYRsnPz6fePCZzVsTEvXt0z\n0eVUOuu2ZVM1OYmMBz6lce3qrN2299LSja9PL7Tv4g07mbosNHHiK9f2VFCIxJjCIkJBME7WuHnr\nS9hTSsuzXyxi9MyfaNuoFmNmrQ2vjwyKf990IjNWbOWBD+cCMOe+s8nNc17+dildW9Tl9KMbx71u\nkcpGYRFhz5iKBRVjbMVy4dVvl7N2Wzbz1m4vtP4/vzmZacs2c26XZjRLq8Fxrepz5fHpJCcZKVWS\noRrcepbuehKJF4VFBGVE/Lg7V788lbXbsunWsi79jmnC0U1q07t9I3Zk51GvVjW6taxb6Jia1fTP\nVSRR9L8vQkUZrr08WLR+BxMWbADg7vM7FepzqFdLo8CKlDVJiS6gLFFUxEdufgHvz/wJgMcv7arO\naZFyQGERQQ2L+Bg5bRX/N34RAOd2aZbgakQkGgqLCLoMFR/PT1gMwA292+p5FpFyQmERQVkRexu2\n72b5pl2YwZ/O6ZjockQkSgqLCJFZka/7Z2Ni5ZZdADxycZcEVyIiB0NhEcjKyeeFr5aEl3fszktg\nNRXDqi27yMrJL7RuxoqtQGhCIhEpPxQWgafGLeSfXywOL2fuyk1gNWVXfoHz+dx17M7LP+B+27Nz\nOeWR8XS8+2PGzFqDu7M2M5spSzcB0LJ+jXiUKyKlRM9ZBHbnFhRa3rRzN+kNaiaomrJp3bZs+jz+\nJTt251G/VjVeujqD7sFtr+u3ZzNjxVY+/nEt/To14aZg7msgPA/2Hu2bpOoBO5FyRv9jAzWrFb4r\nZ/POnARVUnZ9uWBD+PJcdm4+P//nt/z61Lbc1rc9d4z8gS+Dh+xGzVgdPmb+A/15e+pK7n5/NgBJ\nBse10nMVIuWNwiJQo0hYbNqhsNhj2rLNDH1/NvPWbgNCA/u1a1Kbh8bMY9iEJXw6Zx2rgo7rszo2\nplHtFFZtyeL+AceSUiWZq05szbmdm7EmM5ujGqeSpPmwRcodhUWgaMti487dCaokvgoKnOkrtvD8\nl0v4Y/+jad+kdqHtOXkFXPLcxPBy0zrVOa5VfQAeurgz53Vpxl2jZpGb71x3ShuGntep2O/TIDWF\nBqkpsTsREYkphUVgn8tQlaRl8cRnC3hqXOhp6q8WbuDP53VibWYWtVKqcP2pbfnrmLnhfYf98jg6\nNqtT6PiTj2rI2Ft7M2rGak45qmFcaxeR+FFYBIo+SbypEvRZfLt4YzgoGtdOoUOzOgz9z4/h7cO/\nXsrGIDS//0s/0mpULfZ9qldN5ope6bEvWEQSRrfOBqomF/5RbNxRMS9DLd24k37/+JLXJi3nyhcm\nA3DdKW2Y8MczeOWangwJnqruf0zTcFBc3L35foNCRCoHtSz2o6J1cGfn5pOVk89jY+exYN2OQi2I\nP5/bEQs6na/v3ZZrTm5NlSRj444cPpu7jssyWiaqbBEpIxQWxUhOMtZvzy55x3Lk6XELeWHCUnLy\nQ8+TXH9qG96eupIPf3dqOCj22NPKalQ7RZeXRASI8WUoM+tvZvPNbJGZ3VnM9hQzezvYPtnMWgfr\nW5tZlpnNDL6ei2WdUHgQwRb1arBxRw7ZuQd+Srk8eWb84nBQXHJcC4ac24kf7jmblvX14KGIlCxm\nLQszSwaeAfoCq4CpZjba3edE7HYdsMXdjzKzgcAjwOXBtsXu3i1W9RXlEcMItqhXg+WbdrEmM5s2\nDWvFq4RSNXLaSiYs3EiX5mn888vQMCbVqybxt0u70rWFxmUSkYMTy5ZFL2CRuy9x9xzgLWBAkX0G\nAP8KXr8L9LGi10QSoHnd0LhFq7dkxeX7bdqxm49/XHPY82ms35bN8k07AfjHpwv44PufeHDM3PDT\n6EPO7cR5XY5Qa0JEDlosw6I5sDJieVWwrth93D0PyAQaBNvamNkMM/vSzE6NYZ2Evv/e1y3qhT5M\nf9oan7AYOX0VN77+XaFRbw/FvR/Moc/jX/LQmLn8lBnqczmyUS06NavDM1f24Oc9iv74RUSiU1Y7\nuNcA6e6+ycyOA/5jZse4+7bIncxsMDAYID299Dpim6ZVxwxWlUJYLN6wg49mreHm048iKan4RtPW\nYITbv46ZR8t6NflZ5+KnGp2+fDMpVZI5tnnaPtvcnQ9nrQHg+Qmh0PnL+Z345QmtyMkv0MB9InJY\nYtmyWA1E3nPZIlhX7D5mVgVIAza5+2533wTg7tOBxUD7ot/A3Ye5e4a7ZzRq1Oiwio28AFQ12Wha\np3p4vKPD8eEPa/jbJwt4bdLyYrfv2J3Hc0GfwnGt6nHr2zOZumxzsfsOGfUjFz/7Lf+JGKgvL7+A\nNyYv57Z3vgdCLYm7zulAy/o1+NmxzaiSnKSgEJHDFstPkalAOzNrQygUBgJXFtlnNHA1MBG4BBjn\n7m5mjYDN7p5vZm2BdsDhXaMpQWR/wY7sPNo2qsXiDTsP+f2ycvK58sVJJAddMI98PI8zOzTep79g\nU8TDf8N+eRyXPj+Ra4ZP4dXreoXHYAJ4deIy5q3dDsCtb89k6cadNK6TwiMfzaN61WTWbw+9z61n\ntef8rkcwuPeRh1y7iEhRMQsLd88zs1uAsUAyMNzdZ5vZfcA0dx8NvAS8ZmaLgM2EAgWgN3CfmeUC\nBcCN7l78r9sxkJmVS7vGtXl3+ircfZ/nEKKxJjMrPCtc1WQjyYw73v2eN359At8u3siCdTv41cmt\n+WT2OgCeHdSDBqkpjLj+BAYOm8TVw6fy6nW9yMzK5Zlxi5i1OhOAQcenk51bwJOfL6R2ShW2785j\nW3Ye7Rqn0rxeDXq2rr/fmkREDlVMr0+4+xhgTJF1d0e8zgYuLea4fwP/jmVtB7J1Vy5HNk5lx+48\n1m7Lplnawc/qtueZBoDUlCoMObcTt4/8nic+W8DiDTsYM2stBjwYDNS3ZyDDJnWqM+L6E7h82ER+\n8eJkOjdPY9ryLQDUr1WN2/sdTd2aVWnfJJVHPp5H49op/PKEVnRPr8cp7TSQn4jEhi5mFyPUskgF\nYOG6HQcdFtm5+fzixdC4S11bpJFWsxqXHNeCyUs28fS4Rezp577/w72PnJx4ZIPw66Zp1Rl5w4lc\n/fJUJi/dTPWqSTSvW4MzOzSmXq1qANxw2pFktK7H5p259O3U5HBOV0SkRAqLQOSts2cf05Sj9oTF\n+h30bh9d5/nExZsYNy80ltKeQfhuPas9Z3RoDMB9A45l1upM5q3dTs1qybRrnMr3qzLp2KwOKVUK\nj3rbuE513rnhBG5+4zu2Zecx6qaTKHo1LLJPQ0QklhQWgT1PcI+//XTaNKyFu9MwNYXZQV9BNMbO\nXssr3y4jN39v8rSKmMe7RrVkXrgqg1MfHc+xzdN4dlAPznnyK/rtp2VQu3pVXv1VL/ILfL+33YqI\nxIPCoog9H8lmRvf0usxcuTXqY3ODfopXvl0GwEtXZ9C2UWqhfVrWr8mUu/qQnGQ0SE1h8l19DtiB\nbmZUSVZQiEhiaT6LwJ7LUJGf291a1mXJxp1s3VXycOXj569n2aadVK+690da9NLSHo3rVA9PMVoG\nRjcRESnIah8bAAALJ0lEQVSRwuIAuqeHBtybEUXr4tqXp/LNok00qJXChd2OAKBWSvFhISJS3igs\nAuGWBXt/0+/aoi5Vk41JSzZF/T6rt2bx+GXdeHvwCXRrqdFdRaRiUFgEihvvtVZKFTJa1efL+RsO\nfGzErVSntW9EcpJxfNsGusQkIhWGwqKIop/vpx3diHlrt7Nu2/5nzssrCIXFH/q255Vre8ayPBGR\nhFBYBPY3l8RpwTMW4+at3++xn84JDdlRrUqSWhMiUiEpLAL7m3aoQ9PatGlYi/dnFh0wd6+b3/gO\nCI0gKyJSESksiijaMDAzLuzWnElLNrN6P/NbdG0Rml+i/7FNY12eiEhCKCz2OMCMphd2D90K+87U\nlcVur1YliRPbNuCYI/adlEhEpCJQWBRRXJ9Dqwa16NOhMa9NWk5WTv4+23PznapV9KMUkYpLn3CB\nPWND7a97enDvtmzemcPI6fu2LnLzC6iqsZtEpAJTWAT2czNUWK829enVuj5Pfb6Qbdm54fWrt2Yx\n+6dtBzWGlIhIeaOwKGJ/d76aGUPP68SmnTk8+dnC8Pqfgk7vY5urv0JEKi6FRaCEhgUAnVukcUWv\ndIZ/s5SJi0NDgOwZafbG0zTntYhUXAqLImy/vRYhQ87pSKv6NbntnZms35YdnruiWhX1WYhIxaWw\nCBQ3RHlxaqVU4f+u7EFmVi7XvjKVzKxQ/0WVJP0oRaTi0idcwKO6EBVybPM0nhnUg3lrt/O7ETMA\nNEGRiFRoCosiov3IP+Poxjw7qEd4uWqyfpQiUnHpEy4QvnX2IBoIZx/TlBHXn8C5XZqRXr9myQeI\niJRTmoP7MJ14ZANOPLJBossQEYkptSwCexsW6nsQESlKYbFHSY9wi4hUYgqLIjR3kYjIvhQWgUPo\n3xYRqTQUFiIiUiKFRWDvE9xqW4iIFKWwCLg6uEVE9iumYWFm/c1svpktMrM7i9meYmZvB9snm1nr\niG1/CtbPN7OzY1lnoZri9Y1ERMqRmIWFmSUDzwA/AzoBV5hZpyK7XQdscfejgH8AjwTHdgIGAscA\n/YFng/eLmXAHt9JCRGQfsWxZ9AIWufsSd88B3gIGFNlnAPCv4PW7QB8LdRoMAN5y993uvhRYFLyf\niIgkQCyH+2gORE5YvQo4fn/7uHuemWUCDYL1k4oc2zwWRW7dlcOlz01ky67QUON6gltEZF/lemwo\nMxsMDAZIT08/pPdISjLaNUkF4Ii0GtSpUa5/JCIiMRHLT8bVQMuI5RbBuuL2WWVmVYA0YFOUx+Lu\nw4BhABkZGYd0O1Od6lV5dtBxh3KoiEilEcs+i6lAOzNrY2bVCHVYjy6yz2jg6uD1JcA4D93DOhoY\nGNwt1QZoB0yJYa0iInIAMWtZBH0QtwBjgWRguLvPNrP7gGnuPhp4CXjNzBYBmwkFCsF+7wBzgDzg\nN+6eH6taRUTkwKyiPIyWkZHh06ZNS3QZIiLliplNd/eMkvbTE9wiIlIihYWIiJRIYSEiIiVSWIiI\nSIkUFiIiUqIKczeUmW0Alh/GWzQENpZSOeVBZTtf0DlXFjrng9PK3RuVtFOFCYvDZWbTorl9rKKo\nbOcLOufKQuccG7oMJSIiJVJYiIhIiRQWew1LdAFxVtnOF3TOlYXOOQbUZyEiIiVSy0JEREpUqcLC\nzPqb2XwzW2RmdxazPcXM3g62Tzaz1vGvsnRFcc63mdkcM/vBzD43s1aJqLM0lXTOEfv93MzczMr9\nnTPRnLOZXRb8Xc82szfjXWNpi+LfdrqZjTezGcG/73MSUWdpMbPhZrbezH7cz3Yzs6eCn8cPZtaj\nVAtw90rxRWiY9MVAW6Aa8D3Qqcg+NwPPBa8HAm8nuu44nPMZQM3g9U2V4ZyD/WoDEwhN35uR6Lrj\n8PfcDpgB1AuWGye67jic8zDgpuB1J2BZous+zHPuDfQAftzP9nOAjwADTgAml+b3r0wti17AIndf\n4u45wFvAgCL7DAD+Fbx+F+hjZuV5Uu4Sz9ndx7v7rmBxEqFZCcuzaP6eAe4HHgGy41lcjERzztcD\nz7j7FgB3Xx/nGktbNOfsQJ3gdRrwUxzrK3XuPoHQvD/7MwB41UMmAXXNrFlpff/KFBbNgZURy6uC\ndcXu4+55QCbQIC7VxUY05xzpOkK/mZRnJZ5z0Dxv6e4fxrOwGIrm77k90N7MvjGzSWbWP27VxUY0\n53wP8AszWwWMAX4bn9IS5mD/vx+UWM7BLeWImf0CyABOS3QtsWRmScDfgWsSXEq8VSF0Kep0Qq3H\nCWbW2d23JrSq2LoCeMXdHzezEwnNynmsuxckurDyqDK1LFYDLSOWWwTrit3HzKoQarpuikt1sRHN\nOWNmZwFDgAvcfXecaouVks65NnAs8IWZLSN0bXd0Oe/kjubveRUw2t1z3X0psIBQeJRX0ZzzdcA7\nAO4+EahOaAyliiqq/++HqjKFxVSgnZm1MbNqhDqwRxfZZzRwdfD6EmCcBz1H5VSJ52xm3YHnCQVF\neb+ODSWcs7tnuntDd2/t7q0J9dNc4O7leU7eaP5t/4dQqwIza0jostSSeBZZyqI55xVAHwAz60go\nLDbEtcr4Gg1cFdwVdQKQ6e5rSuvNK81lKHfPM7NbgLGE7qQY7u6zzew+YJq7jwZeItRUXUSoI2lg\n4io+fFGe82NAKjAy6Mtf4e4XJKzowxTlOVcoUZ7zWKCfmc0B8oE73L3ctpqjPOc/AC+Y2e8JdXZf\nU55/+TOzEYQCv2HQD/MXoCqAuz9HqF/mHGARsAu4tlS/fzn+2YmISJxUpstQIiJyiBQWIiJSIoWF\niIiUSGEhIiIlUliIiEiJFBZSqZhZIzP72sx+NLMLI9a/b2ZHHOR77SjFum40s6uC1x3MbGYwWuqR\nZvbtQb7XPWZ2e2nVJgIKC6l8rgCeIzQQ3a0AZnY+MMPdEzbQnLs/5+6vBosXAu+6e3d3X+zuJyWq\nLpE9FBZS2eQCNYEUID8Y1uVW4NH9HWBmTcxslJl9H3ydVGR7ajAXyHdmNsvMBgTra5nZh8ExP5rZ\n5cH6h23vHCJ/C9bdY2a3B3Mu3ArcZGbjg207Ir7XHWY2NTj23oj1Q8xsgZl9DRxdOj8qkb0qzRPc\nIoE3g6/BwP8SmsPktYhh2ovzFPClu19kZsmEnniPlA1c5O7bgqE0JpnZaKA/8JO7nwtgZmlm1gC4\nCOjg7m5mdSPfyN3HmNlzwA53/1vkNjPrR2g8p16E5iwYbWa9gZ2ERhvoRuj/9HfA9IP7sYgcmMJC\nKhV3zwT2fHjXA+4ELjKzF4B6wOPBoHORzgSuCo7PJzR0fSQD/hp8cBcQGha6CTALeNzMHgH+6+5f\nBS2ZbOAlM/sv8N+DKL9f8DUjWE4lFB61gVF7Ai8IKpFSpctQUpkNBR4k1I/xNaFBJO85hPcZBDQC\njnP3bsA6oLq7LyA0s9ks4AEzuzuYJ6UXocm1zgM+PojvY8BD7t4t+DrK3V86hHpFDprCQiolM2sH\ntHD3Lwj1YRQQGmyuRjG7f05oylnMLNnM0opsTwPWu3uumZ0BtAr2PQLY5e6vExqwsYeZpQJp7j4G\n+D3Q9SDKHgv8KngPzKy5mTUmND3shWZWw8xqA+cfxHuKREWXoaSyepDQHB4AIwgN4X0ncHcx+/4P\nMMzMriM0YutNQOSlqjeAD8xsFjANmBes7ww8ZmYFhDrWbyJ0yeh9M6tOqKVwW7QFu/snwVDbE4MR\ngncAv3D378zsbULzUK8nNHy3SKnSqLMiIlIiXYYSEZESKSxERKRECgsRESmRwkJEREqksBARkRIp\nLEREpEQKCxERKZHCQkRESvT/XEYAMMontokAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c201c45c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(N)/N, error_rate)\n",
    "plt.xlabel('% classified')\n",
    "plt.ylabel('% error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def custom_convolution(a, kernel):\n",
    "    n = len(a)\n",
    "    k = int((len(kernel)-1)/2)\n",
    "    \n",
    "    conv = np.zeros(n)\n",
    "    for i in range(n):\n",
    "        imin_kernel = max(0, k-i)\n",
    "        imax_kernel = min(2*k+1, k+n-i)\n",
    "        imin_a = max(0, i-k)\n",
    "        imax_a = min(n, k+1+i)\n",
    "        conv[i] = (a[imin_a:imax_a]*kernel[imin_kernel:imax_kernel]).sum() / (kernel[imin_kernel:imax_kernel].sum())\n",
    "    \n",
    "    return conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f1c19f9d160>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VGXa//HPlU4JNaGFktB7DUVERaxYwIZ1d8W17Ori\n6rPt2efRddV1f7urz9p2reu6llWxIqgoggqKSAkQeq9JgCS0EEjP3L8/ZnAjJjApk8lkvu/Xixcz\n55w557o5ZK7c5z7nus05h4iIyKlEBDsAEREJDUoYIiLiFyUMERHxixKGiIj4RQlDRET8ooQhIiJ+\nUcIQERG/KGGIiIhflDBERMQvUcEOoLoSEhJccnJysMMQEQkpy5cv3++cS6zNPkIuYSQnJ5OWlhbs\nMEREQoqZ7artPnRJSkRE/KKEISIiflHCEBERvyhhiIiIX5QwRETEL0oYIiLiFyUMERHxixKGiEgD\n5/E4/vjRerZk5wc1DiUMEZEGbvbavfzjqx2szswLahxKGCIiDVhpuYdH5myib4d4LhuWFNRYlDBE\nRBqw15fsZteBAn5zYR8iIyyosShhiIg0UIcLSnhs3mbG9Uzg7D7tgh2OEoaISEP1+LwtHCks5d5L\n+mEW3N4FhGC1WhGRhq64rJzXFu9mW+5Rzu7TjnP6tav2F/6W7HxeXbyL60Z1pW+HFgGKtHqUMERE\n6lBZuYfb/72Czzfm0DQmkteW7Gbq2GR+f2l/v5OGx+O4Z8ZamsdG8Yvzegc4Yv/pkpSISB16adFO\nPt+Ywx8mD2D178/n5nEpvLRoJ88s2Ob3Pt5My2DpzoPcc1E/2jaPDWC01aMehohIHckrLOWJeVs4\nu08iPzwtGYB7L+5HTn4xj8zZRL+OLU45eL37QAH/76MNjOnehimpneshav+phyEiUkfeTssgv7iM\nX57f59tlZsbDVw6mb4cW3D09nYyDBVV+vrisnGlvrMAMHrlqSIMY6K5ICUNEpA54PI6Xv9nJqOQ2\nDExq+Z11TWIiefYHw3HO8dN/L6eotLzSz//Pe2tYnZnHI1OG0KVN03qK3H9KGCIidWDZzoNkHCzk\nhjFdK13frW0zHr92KOv2HPle0igqLed/3lvDeyuy+MV5vblgQIf6CrtaNIYhIlIHPly9l7joCM7t\n177KbSb0bc+frhjE/85Yw0VPfsXUsck4B68u3sXWnKPcOaEnd07oWY9RV48ShohILZV7HB+v3cuE\nvu1oFnvyr9XrRnWlc+smPPDBeu6buQ6A3u2b8+LUVCb0rTrZNARKGCIitbRk+wH2Hy3hksGd/Nr+\njF6JzP2vM8k8VEhkhNGxZVyDG+CujBKGiEgtfbhmL01jIqtV78nMGuTA9slo0FtEpBbKyj3MWbuP\nCX3b0SQmMtjhBJQShohILSzdeZADx0q4aFDHYIcScEoYIiK18MnafcRFRzC+T2KwQwk4JQwRkRry\neByfrN3H+N7taBrT+IeElTBERGpoZcYhcvKLmTioYT5oV9caf0oUEakg42AB6/bkER0ZwZAurUio\nRTXYj1bvIyYyggl9gz8bXn0IaMIwswuBJ4BI4AXn3J9PWD8VeATI8i36u3PuhUDGJCLhKedIEb+b\nuZY567K/XRYZYVwyuCN3ndOL7onNq7W/kjIPM9OzmNC3HfFx0XUdboMUsIRhZpHAU8B5QCawzMxm\nOefWn7Dpm865aYGKQ0Rk2c6D3P7vFRwrLuPn5/Ti3H7tKC7z3g77xtLdfLR6LzeOTebnE3rRsql/\nX/6fb8zmwLESrhnZJcDRNxyB7GGMArY657YDmNl0YDJwYsIQEQmYpTsOcuOLS+nYMo7Xbx1N7/bx\n364bmdyGn5zVg0fnbubFr3fw3opM7jqnF9eP7kZM1MmHeF9fmkH7FrGc0Ssh0E1oMAI56J0EZFR4\nn+lbdqIrzWy1mb1jZuGTqkUk4FbuPsRN/1pKp1ZxvPmT076TLI5LjI/lT1cM4qM7z6Bfxxbc/8F6\nzntsAR+u3oNzrtL9rs3K48vNufxwTDeiIsPn3qFgt/QDINk5NxiYC7xc2UZmdpuZpZlZWm5ubr0G\nKCKhKetwIbe+kkbb5rG8fusYEuNPPrjdv1MLXrtlNC/dNJIm0ZFMe30llz31NUu2H/jOdh6P44EP\n1tGySTQ/GpscwBY0PIFMGFlAxR5DZ/4zuA2Ac+6Ac67Y9/YFYERlO3LOPe+cS3XOpSYmNv6HY0Sk\ndo4Vl3HLy2kUl3l4cepI2reI8+tzZsb4Pu346Odn8MhVg8nJL+aa5xfzs9dXsC33KKXlHv44ewPL\ndh7inov70SJMBruPC+QYxjKgl5ml4E0U1wLXV9zAzDo65/b63k4CNgQwHhEJE/fPWsemfUd4cepI\nerar3t1P4L17akpqFy4d0olnF2zjmfnb+Gj1XqIijDKPY+rYZKaMaFjzbdeHgCUM51yZmU0D5uC9\nrfZF59w6M3sQSHPOzQJ+bmaTgDLgIDA1UPGISHj4bEM2by/P5Gdn92B8NarHViYuOpK7z+3NdaO6\n8snafew5XMhpPdpyVu/EkChHXtesqkGdhio1NdWlpaUFOwwRaYCOFpcx4f/m06ZZDDOnnU5sVOOu\nHlsdZrbcOZdam33oSW8RaTT+8eV2cvKLee6HI5QsAiDYd0mJiNSJnPwi/vHVdi4e1JFhXVsHO5xG\nSQlDRBqFZ+Zvo6TMw68u6BPsUBotJQwRCXl5BaW8uSyDSUM7kZLQLNjhNFpKGCIS8l5buouCknJu\nPaN7sENp1JQwRCSklXscr36zizN6JdCvY4tgh9OoKWGISEhbuHU/e/OKuG5U12CH0ugpYYhISHsr\nLYPWTaM5p194TGIUTEoYIhKyDheUMHddNpcNS9JzF/VACUNEQtbM9D2UlHuYMkIzI9QHJQwRCVlv\npWUwMKkF/TtpsLs+KGGISEham5XHuj1HuDpVvYv6ooQhIiHpneWZxERGMGlIp2CHEjaUMEQk5BSX\nlfN+ehbnD2hPq6YxwQ4nbChhiEjI+WxDDocLSpmiy1H1SglDRELO22kZdGgRx7ieCcEOJawoYYhI\nSMk5UsSCzblcMTyJyIjwm/UumJQwRCSkzFiZhcfBVWE4p3awKWGISMhwzvH28kxGdGtN98TmwQ4n\n7ChhiEjIWJWZx9aco+pdBIkShoiEjDeXZRAXHcHFgzsGO5SwpIQhIiGhoKSMD1bt4eJBnWgRFx3s\ncMKSEoaIhISPVu/laHEZ14zUsxfBooQhIiHhrbQMuic0Y2Ry62CHEraUMESkwduWe5RlOw8xJbUL\nZnr2IliUMESkwXsrLYPICOPKEUnBDiWsKWGISINWWu7h3eVZTOjbjnbxccEOJ6wFNGGY2YVmtsnM\ntprZb0+y3ZVm5swsNZDxiEjo+XxjDvuPFnONCg0GXcAShplFAk8BE4H+wHVm1r+S7eKBu4AlgYpF\nRELXu8szSYyPZXyfxGCHEvYC2cMYBWx1zm13zpUA04HJlWz3B+AvQFEAYxGREJRXUMr8TblcOrgT\nUZG6gh5sgTwDSUBGhfeZvmXfMrPhQBfn3EcBjENEQtTHa/dSUu7hsmGaVa8hCFrKNrMI4FHgl35s\ne5uZpZlZWm5ubuCDE5EG4f30LLonNGNQUstghyIENmFkARVHqTr7lh0XDwwE5pvZTmAMMKuygW/n\n3PPOuVTnXGpioq5jioSDvXmFLNlxkMlDk/TsRQMRyISxDOhlZilmFgNcC8w6vtI5l+ecS3DOJTvn\nkoHFwCTnXFoAYxKREDErfQ/OweShuhzVUAQsYTjnyoBpwBxgA/CWc26dmT1oZpMCdVwRaRxmpu9h\nSJdWJCc0C3Yo4hMVyJ0752YDs09Ydl8V244PZCwiEjq2ZOezfu8Rfn/p9+7ElyDSfWoi0uC8n55F\nZIRxyWBdjmpIlDBEpEFxzjEzfQ+n90wgMT422OFIBUoYItKgLN91iMxDhVymwe4GRwlDRBqU99Oz\niIuO4PwBHYIdipxACUNEGozCknJmpu/hggEdaB4b0HtypAaUMESkwfhg9R7yi8q4flTXYIcilVDC\nEJEGwTnHa4t30bNdc0altAl2OFIJ9flEJOBmrMzkvRVZdGgRxy1ndKdPh/jvbbNgcy6rMvP44+UD\nVQqkgVIPQ0QC6un5W/mvN1eReaiQT9bu4+Inv+KfC3fgnPt2m3KP46+fbiapVROmjNBESQ2VEoaI\nBMyazDz++ulmLhnckXm/OIsvf3M2E/q24w8frueO11ZwpKgUgL99voU1WXn85sI+xETpa6mh0iUp\nEQmYh+dspFWTaP54+SAiI4zWzWJ47ocjeOGrHfz5k42sfvwrkhOa8vXWA1w+LIlJQ/TsRUOmVC4i\nAbEq4zBfbdnPbWd2p2WT6G+Xmxm3ntmdN24dQ8eWcezNK+IX5/Xm4asGa+yigVMPQ0QC4r0VmcRG\nRXD96MpvkR2V0oZ3bh9bz1FJbaiHISJ1rtzjmL12H2f3aUd8XPSpPyAhQQlDROrckh0HyM0v5pIh\nHYMditShkyYMM4s0s431FYyINA4frd5Lk+hIJvRtF+xQpA6dNGE458qBTWam5/RFxC9l5R4+WbuP\nc/q1o2mMhkkbE3/OZmtgnZktBY4dX+ic0zSrIvI932w/wIFjJZr8qBHyJ2H8LuBRiEij8eGqvTSP\njWJ8n8RghyJ17JQJwzm3wMzaAyN9i5Y653ICG5aIhKKSMg+frNvHef3bExcdGexwpI6d8i4pM7sa\nWApMAa4GlpjZVYEOTERCz9fb9pNXWMrFg3R3VGPkzyWpe4CRx3sVZpYIzAPeCWRgIhJ6Ply1l/i4\nKM7onRDsUCQA/HkOI+KES1AH/PyciISRotJyPl2/jwsGdCA2SpejGiN/ehifmNkc4A3f+2uA2YEL\nSURC0fxNueQXlamAYCPmz6D3r83sCmCcb9HzzrkZgQ1LRELNzPQsEprHMrZH22CHIgFy0oRhZpHA\nPOfc2cB79ROSiISavMJSPtuYw/WjuhIVqSvWjdVJE4ZzrtzMPGbW0jmXV19BiUjweTyOlxbt5O3l\nmTSLieTmcSlMrOLup7eWZVBS5uGqEZ3rOUqpT/78KnAUWGNm/zSzJ4//8WfnZnahmW0ys61m9ttK\n1v/UzNaYWbqZLTSz/tVtgIjUPecc93+wjgc/XE+zmEgOFZRw+2sr+PPHG78ztSp4S4G8tGgno1La\nMDCpZZAilvrgz6D3e9TgcpTvctZTwHlAJrDMzGY559ZX2Ox159yzvu0nAY8CF1b3WCJSt+asy+aV\nb3Zxy7gU7rm4H+UebwJ5dsE28gpLeeiygURGeCc7emPpbrIOF3L/pAFBjloCzZ8xjPOdczfUYN+j\ngK3Oue2+fU0HJgPfJgzn3JEK2zcDvvuri4jUu2PFZdw/ax39Orbgvyf2xcyIijT+MHkgLZtE89QX\n2zhSWMqfrxzErgMFPPzJJsb2aMu5/VSZtrHzZwyjm5nFOOdKqrnvJCCjwvtMYPSJG5nZz4BfADHA\nhGoeQ0Tq2BtLd7PvSBFP3TCM6AoD2GbGry/oS8sm0fzp443M25BNabmHdvFx/N+UIZpeNQz4c0lq\nO/C1mc3iu9VqH62LAJxzTwFPmdn1wL3AjSduY2a3AbcBdO2qSusigVJa7uFfX+9kdEobRnRrU+k2\nt53ZgzHd2/L+yj00jYlk6unJJDSPredIJRj8SRjbfH8igPhq7DsL6FLhfWffsqpMB56pbIVz7nng\neYDU1FRdthIJkNlr9pJ1uJAHJ598PGJw51YM7tyqnqKShsKfB/ceADCzps65gmrsexnQy8xS8CaK\na4HrK25gZr2cc1t8by8GtiAiQeGc4x9fbadHYjPO7qPxCPk+f6rVnmZm64GNvvdDzOzpU33OOVcG\nTAPmABuAt5xz68zsQd8dUQDTzGydmaXjHcf43uUoEakf32w7wNqsI9x6RnciIjQeId/nzyWpx4EL\ngFkAzrlVZnamPzt3zs3mhLpTzrn7Kry+y/9QRSSQnv9qOwnNY7hsWFKwQ5EGyq9n+J1zGScsKg9A\nLCISJBv3HWH+plxuPC1ZEx9JlfzpYWSY2VjAmVk0cBfeS0wi0kg8/+V2mkRH8sPTugU7FGnA/Olh\n/BT4Gd7nKrKAob73ItII7DlcyKz0PVw7qgutmsYEOxxpwPy5S2o/UJMnvUUkBDw6dzMRZtxyRvdg\nhyINnOoQi4Sx1ZmHeXdFJlNPTyapVZNghyMNnBKGSJg6UlTK3dPTaR8fx8/O7hnscCQE+DPoLSKN\nQEmZh+PlnlZlHOb+D9aRcaiAV348mpZNooMbnIQEvxOGmY0B7gfigMedc+8HKigR8fYAXlu8m/1H\ni7l8WFKN5prYkp3PE59tYeHW/RwuKP3OulZNo3nuhyM4TVOqip+qTBhm1sE5t6/Col8AlwMGLAGU\nMEQCJK+glCnPLWJz9lGiI42XFu3k8WuGcumQTn7vY/aavfzXm+nEREUwcWAHurRuihmUe6Bb26ac\n1789zWJ1kUH8d7L/Lc+a2QrgYedcEXAYuArwAEdO8jkRqaX/fX8N23OP8e+bRzOoc0tufTmNX729\nit7t4+nT4dQ1QBdt28+011cwrGtrnv3BCBLjVU1Waq/KQW/n3GXASuBDM/sRcDcQC7QFLquf8ETC\nz9db9/PR6r3cdU4vxvVK8E5adMNw4uOi+fkbKykuO3mhhdz8Yu6ank5KQjNe+fEoJQupMye9S8o5\n9wHeOlItgRnAZufck8653PoITiTcOOd4+JONJLVqwq1n/ue5iMT4WB6ZMphN2fn87bOtVX6+3OP4\nrzfTyS8q5ekbRuiSk9SpKhOGmU0ysy+AT4C1wDXAZDObbmY96itAkXCyZMdBVmXmccfZPb5X0+ns\nPu2YMqIzzyzYxprMvEo//9QXW1m4dT8PTBrg16Urkeo4WQ/jIWAicDXwF+fcYefcL4HfAX+sj+BE\nws2LC3fQumk0Vw7vXOn6ey/pT0LzGH719ipKyjzfWbdo634en7eZy4clcXVql0o/L1IbJ0sYecAV\nwJVAzvGFzrktzrlrAx2YSLjZdeAYczdkc8PoblVWjG3ZJJo/XTGITdn5PDZv87fLt+ce5Y7XV9A9\nsTkPXTZQ82tLQJzsAuflwHVAKSfMlCcide+lRTuJNDtlxdgJfdtz7cguPDN/G8eKy+jVPp4n5nnr\nQf3zxlSNW0jAVPk/y1d08G/1GItI2DpaXMY7aZlcNKgj7VvEnXL7hy4bSGxUBK8s3oVz0LdDPH+/\nfjjd2jarh2glXOlXEZEG4L0VmeQXlzH19GS/to+KjOCByQOZNqEX+UWlJLdtpmlVJeCUMESCzDnH\ny4t2MqRzS4Z1aVWtzybGx+o5C6k3qlYrEmRfbMphW+4xbhybrMFqadCUMESCyDnHY3O30KVNk2rV\niRIJBiUMkSCauz6bNVl53DmhF9GR+nGUhk3/Q0WCpKCkjIc+2kBKQjOuGJYU7HBETkmD3iJ1xDnH\n4YJSCkrLiYuKoHXTmCrvXHLOcd/Mdew+WMD028YQpd6FhAAlDJFa8Hgcc9bt490VWSzZfoD84rJv\n1zWLiaR/pxakJrdhVEobUru1Jj4umvyiUv7f7I28szyTn0/oyZjumsBIQoMShkgN7c0r5O7p6SzZ\ncZBOLeO4dGgneiQ2p3lsJIUl5ew8UMCqzMP848vtPDN/GxEGSa2bkH2kmJIyD7eP78F/ndc72M0Q\n8ZsShkgNZB4q4NrnF3PoWAl/uXIQV43oQmQVl58KS8pZsfsQS3YcZMf+Y3RoEcukIUkM6lz9KVdF\ngimgCcPMLgSeACKBF5xzfz5h/S+AW4AyIBf4sXNuVyBjEqmtotJybn1lOXmFpUy/7bRTfvE3iYnk\n9J4JnN4zoZ4iFAmMgI20mVkk8BTeEun9gevMrP8Jm60EUp1zg4F3gIcDFY9IXXlkziY27D3CE9cO\nVS9Bwkogb80YBWx1zm13zpUA04HJFTdwzn3hnCvwvV0MVD4JgEgDsSU7n5cW7eS6UV2Z0Ld9sMMR\nqVeBTBhJQEaF95m+ZVW5Gfg4gPGI1NpDH22gWUwkv76gT7BDEal3DWLQ28x+AKQCZ1Wx/jbgNoCu\nXbvWY2TSkG3Ozue+mWvZkn2UUSltuOfifnRu3TRgx1uTmceCzbn894V9adMsJmDHEWmoAtnDyAIq\nzhPZ2bfsO8zsXOAeYJJzrriyHTnnnnfOpTrnUhMTEwMSrISWrTlHuea5b9iSfZSz+iTy5eZcrnxm\nERkHC0794Rp6dsE24uOi+MEY/dIi4SmQCWMZ0MvMUswsBrgWmFVxAzMbBjyHN1nkVLIPke8p9zh+\n+fYqAN69fSyPXj2Ud+8YS0FJOdNeX/G9ua7rwo79x5i9di8/GNON+LjoOt+/SCgIWMJwzpUB04A5\nwAbgLefcOjN70Mwm+TZ7BGgOvG1m6WY2q4rdiXxrxsosVmUc5v5JA0hO8M4w17dDCx6+cjCrMvP4\n58IddX7M57/cTnRkBDf5OcGRSGMU0DEM59xsYPYJy+6r8PrcQB5fGp9yj+Pp+Vvp17EFk04oBz5x\nUEfO79+eJz/bwuShnejUqkmdHPPgsRLeW5HJlcOTaBd/6ulTRRorVTyTkPLFxhy25x7jjvE9Kp1s\n6L5L++Nw/OnjjXV2zDeW7qa4zMOPT0+ps32KhCIlDAkpb6VlkNA8lokDO1S6vnPrptx2Zg8+WLWH\n5bsO1vp4JWUeXl60kzN6JdCrfXyt9ycSypQwJGTk5hfz+cYcrhyedNJy4D89qzvtW8Ty4Icb8Hhc\nrY45e81ecvKL+fE49S5ElDAkoHLyi9h14Fid7GtmehZlHseU1JMXBGgaE8WvL+jLqozDzFq1p8bH\nc87x4tc76J7YjLN66XZuESUMCZgZKzM5/c+fc9Yj87lnxhqcq91v+x+t2cvApBb0bHfqS0NXDEti\nUFJL/vLJRgpLymt0vK+27Gd1Zh43j0upciIkkXCihCEBsTUnn/9+Zw3Du7bmR6d147Ulu3ljacap\nP1iFfXlFrNx9mIkDO/q1fUSE8btL+rM3r4jnv9xe7eM553jisy10ahnHVSNU4kwElDAkQB6bt4Xo\nSOOpG4bzwKQBjEppwyNzNnK0wox01TFn3T4ALhhQ+WB3ZUaltOHiQR15dsE2Mg9V7wnwLzblsHzX\nIW4/uyexUZHV+qxIY6WEIXVuz+FCZq/Zyw9PSyaheSxmxm8n9uVQQSnvpNWsl/HJ2n30bNecnu2a\nV+tzv53Yl8gI4/Z/r6Co1L9LU0Wl5TzwwXq6JzbjmtQup/6ASJhQwpA6N2NlFs7BdaP+82U7vGtr\nhnVtxb8W7az2nUsHj5WwZMcBLqxG7+K4Lm2a8tg1Q1mTlcdd01dSXHbqpPGn2RvYdaCABycNJCZK\nPyIix+mnQeqUc473VmQyMrk13do2+866qWOT2XWggMU7DlRrn/PWZ+NxcGEVz16cynn92/P7S/sz\nZ1021/9jCVtzjlYZ+xPztvDyN7u4eVwK43pphjyRihpEeXNpPLbvP8a23GNMHZv8vXUXDOhA89go\nZqzIYmwP/7+MP167l86tmzCgU4sax3XT6SkkNI/lnhlrOP+xBZzVO5GzeifSp0ML4uOi2HngGNOX\nZrBw636uHN6Z/72oX42PJdJYKWFInVqwKReA8X3afW9dXHQkEwd24OO1+3hw8kCaxJx6MPlIUSkL\nt+5n6tjkSkuBVMelQzoxpntbXl60k/fTs/jCF+tx7eJj+d0l/blpbLJuoxWphBKG1KkFm3PpntiM\nLm0qn8jo8uFJvL08k7kbsr9XPLAyn23IprTcMXGQf7fTnkpifCy/uqAPv7qgDxkHC9h9sID8olLa\ntYhjcFLLkz5BLhLulDCkzhSVlrN4+wGuH131BENjUtrSsWUcM1Zk+pUwZq/ZR4cWcQzt3KouQwW8\nA+JVJTYR+T79OiV1ZsmOgxSXeTird9VlNCIijMuGJfHllv3k5BeddH9Hi8tYsDmXCwd20CUikQZA\nCUPqzIJNucRGRTCme9uTbnfl8M6UexwzV568ztPnG3MoKfNwUR1djhKR2lHCkDqzYHMOo7u3JS76\n5IPZPds1Z2iXVryzPPOk9aVmrMikfYtYRnRrXdehikgNKGFIncg4WMC23GOc6eezC1eN6Mym7HzW\n7TlS6fp9eUUs2JzLVSM6E6nLUSINghKG1Ikvtxy/nda/MuCXDu5ETFQEby6rvFTIuysy8Ti4aoRK\nc4g0FEoYUicWbMolqVUTeiT6V+upZdNoLh3cibeXZ3DgaPF31hWXlfPKNzs5rXtbUhKaVb4DEal3\nShhSa6XlHhZtO8CZvROr9XDd7eO7U1zm4YWFO76z/N3lWWQfKeb28T3qOlQRqQUlDKm1ZTsPcrS4\n7KS301amZ7t4Jg/pxAtfbWdzdj4A+48W88icjaR2a80ZquUk0qAoYUitzd+US3Sk1ahY372X9Cc+\nLpqbX17GzPQsbvrXMgpKynno8oG1LgUiInVLCUNq7YuNOYxOaUvz2OoXDkhoHsuLU0dSXOrhrunp\n7Nx/jKeuH07fDjUvNCgigaHSIFIrGQcL2JJzlGtG1vxupqFdWjH/1+PZtC+f7onNadkkug4jFJG6\nooQhtTJ/s/d22rP7fr86bXU0jYliWFc9oCfSkOmSlNTKZxuy6dqmKd11+6tIoxfQhGFmF5rZJjPb\nama/rWT9mWa2wszKzOyqQMYide/QsRIWbtnPxEEdNEAtEgYCljDMLBJ4CpgI9AeuM7P+J2y2G5gK\nvB6oOCRwPlm3jzKP49LBpy5TLiKhL5BjGKOArc657QBmNh2YDKw/voFzbqdvnSeAcUiAzEzPontC\ns1pNnSoioSOQl6SSgIqFgjJ9y6QR2JpzlMXbD3LF8CRdjhIJEyFxl5SZ3QbcBtC1a9WzuYW7gpIy\nvty8n8xDBTSNiWJM9zZ097O2U3X9e/EuYiIjuHaUzodIuAhkwsgCKt6c39m3rNqcc88DzwOkpqZW\nPYFCmCop8/DCwu08/cU2jhaXfWfd2X0SufeS/n4XBfRHbn4xb6VlcMngjiQ0j62z/YpIwxbIhLEM\n6GVmKXgvYbbsAAAOB0lEQVQTxbXA9QE8XljKOVLET/+9nBW7D3Nuv/bcPC6F/h1bcLiwhA9W7eG5\nBdu58PEvuWN8T+44uwexUSef3MgfT362heIyD3ee06sOWiAioSJgCcM5V2Zm04A5QCTwonNunZk9\nCKQ552aZ2UhgBtAauNTMHnDODQhUTI1NxsECrn1+MQePlfD364dxSYW7lVo2jWbahF5cM7IrD320\nnic+28LsNXv585WDGNGtzbfbHS0u44uNOXy6Ppt1WXmUejz0aR/PhQM7csngjt+bPW/Rtv38e8ku\nbjwtWaXHRcKMnWyKzIYoNTXVpaWlBTuMoNtzuJCrn/uG/KIyXrtlNAOTWp50+y825XDvjLVkHS5k\nWNdW9Exszp68QpbtOERJuYe2zWIYldKGqMgIVu4+ROahQhLjY5k6NpkbRnelZZNovtiUw93T00mI\nj+XDO8fRNCYkhsBEBDCz5c651FrtQwkj9OQXlXLF04vYl1fEa7eOZnDnVn597lhxGS9/s5M5a/ex\n70gRCc1jGdujLef178CIbq2/nQrVOceibQd47svtfLk5FzOIj43iSFEZfTvE88KNqXRu3TSALRSR\nuqaEEYbKPY7bXklj/uZcXv3xKMb2DOycEev3HOHT9fs4cLSEgUktmDw06XuXqUSk4auLhKFrCiHm\nr59u4rONOTw4eUDAkwVA/04t6K8H80QEFR8MKR+u3sPT87dx3aiu/HBMt2CHIyJhRgkjRGzOzuc3\n76xmRLfWPDBpgJ6uFpF6p4QRAo4UlfKTV5fTLDaKp28YTkyUTpuI1D9989TQvrwiFm3bz968woAe\nx+Nx/PKtVWQcLOCp64fTvkVcQI8nIlIVDXpXU1FpOffPWsf0Zf+pq3huv3bcc3H/gDzI9ujczcxd\nn819l/RnVEqbU39ARCRA1MOohnKPY9rrK5m+LINbxqXw2i2jufvcXizZcZCLnviKV7/ZicdTd7cp\nv7F0N3//YivXjerKTacn19l+RURqQj2Mavjnwu3M25DN/Zf2Z+rpKQCc3jOBa0d25TfvruZ3M9fx\n8dp9/OXKwXRpU7sH22amZ3Hv+2sZ3yeRP0zWILeIBJ96GH7asf8Y/zdnMxcMaM+NY5O/s65Dyzhe\nvmkkf7piEKsz8zjn0QX87v21rM3KoyYPRr6xdDd3v5nOyOTWPHX9cKIidZpEJPjUw/DTo3M3Exlh\n/OGygZX+tm9mXDeqK2f1TuRvn29h+rLdvLp4F51bN+Hcfu2Z0Lcdo7u3OWm12LJyD4/P28Lfv9jK\n+D6JPPuDEXqqWkQaDJUG8cOGvUeY+MRX3DG+B7+5sK9fnzl0rIRP1+9j7vpsFm7dT1Gph2YxkZzR\nK5Fz+7fnnL7taN0sBvDeCbVo2wEenrOR1Zl5TBnRmT9ePki3z4pInVFpkHry1083Ex8XxU/O7OH3\nZ1o3i+GakV25ZmRXikrLWbRtP59tyOGzDTl8sm4fkRFGctumtGwSze6Dhew/WkyHFnH87bphXDqk\n06kPICJSz5QwTiE94zDzNmTzy/N607JpdI32ERcdyYS+7ZnQtz0PXeZYk5XH3PXZbMs9yuGCUs7s\nlcBZfRK5YEAHXYISkQZLCeMU/vrpJto0i+GmcSl1sj8zY3DnVn6XJBcRaSjC6iJ5dcdrlu44yFdb\n9nP7WT1oHqvcKiLhLWwSxvxNOVzz3OJqlfJ4dO4mEuNj+YEqw4qIhE/CKCwpZ92ePC5+ciHzN+Wc\ncvvPNmSzePtB7hjfgyYxGlcQEQmbhDFxUEdm3TmOdvGx3PTSMp6Yt6XKMh6FJeX8ftY6erVrzg2j\n1bsQEYEwShgAPRKbM+OO07l8aBKPzdvMra+kkVdY+p1tnHP8ftZaMg8V8ofLBupZCBERn7D7NmwS\nE8lfrx7Cg5MHsGBzLpP+vpCFW/bjnKOotJw/fLiBt9IymXZ2T8Z0bxvscEVEGoywvPXHzPjRackM\n6NSCn7+Rzg/+uYS2zWIoLvNwtLiMqWOT+cV5vYMdpohIgxKWCeO4Ed3a8PmvzmJW+h6W7jhIbHQE\nFw/qxGk91LMQETlRWCcMgNioSKakdmFKapdghyIi0qCF3RiGiIjUjBKGiIj4JaAJw8wuNLNNZrbV\nzH5byfpYM3vTt36JmSUHMh4REam5gCUMM4sEngImAv2B68ys/wmb3Qwccs71BB4D/hKoeEREpHYC\n2cMYBWx1zm13zpUA04HJJ2wzGXjZ9/od4BzT5NUiIg1SIBNGEpBR4X2mb1ml2zjnyoA8QPe0iog0\nQCEx6G1mt5lZmpml5ebmBjscEZGwFMiEkQVUfLihs29ZpduYWRTQEjhw4o6cc88751Kdc6mJiYkB\nCldERE4mkA/uLQN6mVkK3sRwLXD9CdvMAm4EvgGuAj53p5jlaPny5fvNbFcNY0oA9tfws41BOLc/\nnNsO4d1+td2r1qW3A5YwnHNlZjYNmANEAi8659aZ2YNAmnNuFvBP4FUz2wocxJtUTrXfGncxzCzN\nOZda08+HunBufzi3HcK7/Wp73bU9oKVBnHOzgdknLLuvwusiYEogYxARkboREoPeIiISfOGWMJ4P\ndgBBFs7tD+e2Q3i3X22vI3aKMWYREREg/HoYIiJSQ2GTME5VCLExMLOdZrbGzNLNLM23rI2ZzTWz\nLb6/W/uWm5k96fv3WG1mw4MbffWZ2YtmlmNmayssq3Z7zexG3/ZbzOzGYLSluqpo+/1mluU7/+lm\ndlGFdf/ja/smM7ugwvKQ+7kwsy5m9oWZrTezdWZ2l295uJz7qtof+PPvnGv0f/De1rsN6A7EAKuA\n/sGOKwDt3AkknLDsYeC3vte/Bf7ie30R8DFgwBhgSbDjr0F7zwSGA2tr2l6gDbDd93dr3+vWwW5b\nDdt+P/CrSrbt7/s/Hwuk+H4WIkP15wLoCAz3vY4HNvvaGC7nvqr2B/z8h0sPw59CiI1VxQKPLwOX\nVVj+ivNaDLQys47BCLCmnHNf4n1+p6LqtvcCYK5z7qBz7hAwF7gw8NHXThVtr8pkYLpzrtg5twPY\nivdnIiR/Lpxze51zK3yv84ENeOvShcu5r6r9Vamz8x8uCcOfQoiNgQM+NbPlZnabb1l759xe3+t9\nQHvf68b6b1Ld9ja2f4dpvssuLx6/JEMjbrtvDp1hwBLC8Nyf0H4I8PkPl4QRLsY554bjnYPkZ2Z2\nZsWVzts/DZvb4sKtvcAzQA9gKLAX+GtwwwksM2sOvAvc7Zw7UnFdOJz7Stof8PMfLgnDn0KIIc85\nl+X7OweYgbfLmX38UpPv7xzf5o3136S67W00/w7OuWznXLlzzgP8A+/5h0bYdjOLxvtl+Zpz7j3f\n4rA595W1vz7Of7gkjG8LIZpZDN6aVbOCHFOdMrNmZhZ//DVwPrCW/xR4xPf3TN/rWcCPfHeQjAHy\nKnTnQ1l12zsHON/MWvu68Of7loWcE8agLsd7/sHb9mvNOyVyCtALWEqI/lyYmeGtQ7fBOfdohVVh\nce6ran+9nP9gj/jX1x+8d0psxntXwD3BjicA7euO9y6HVcC6423EOyHVZ8AWYB7Qxrfc8E6huw1Y\nA6QGuw01aPMbeLvepXivv95ck/YCP8Y7ELgVuCnY7apF21/1tW217we/Y4Xt7/G1fRMwscLykPu5\nAMbhvdy0Gkj3/bkojM59Ve0P+PnXk94iIuKXcLkkJSIitaSEISIiflHCEBERvyhhiIiIX5QwRETE\nL0oYEpbM7BFfpc9HzOynZvajSrZJrlgNNpT4Kpf+KthxSOMS0Dm9RRqw2/Dep18e7EBOxveQljnv\n07siQaUehoQUM/uRr7jaKjN71bcs2cw+9y3/zMy6+pa/5JsHYZGZbTezq3zLZwHNgeVmdk3F38bN\nbIRv36uAn1U4bqSvN7LMd5yf+JaPN7P5ZvaOmW00s9d8X/KY2UjfsVeZ2VIzi69qPye0Mdk3R8Er\neJ/W7WJmz5hZmq9X9ECFbXea2QNmtsK8c6H0rWR/t5rZx2bWpM5OhIQlJQwJGWY2ALgXmOCcGwLc\n5Vv1N+Bl59xg4DXgyQof64j3ydhLgD8DOOcmAYXOuaHOuTdPOMy/gDt9+6/oZrwlJUYCI4FbfWUW\nwFst9G688w50B073lVp4E7jLt69zgcJT7KeiXsDTzrkBzrldeJ/CTQUGA2eZ2eAK2+533qKTzwDf\nuQxlZtN8bb/MOVdYyXFE/KaEIaFkAvC2c24/gHPu+HwQpwGv+16/ijdBHPe+c87jnFvPf8pdV8rM\nWgGtnHeuieP7Ou58vPWI0vGWkm6L90sdYKlzLtN32SgdSAb6AHudc8t8sR5xzpWdYj8V7XLeuRuO\nu9rMVgArgQF4k9Nxx4vvLfcd+7gf4a1cfJVzrvhkbRfxh8YwpLGr+EVptdiP4e15fKc4nZmNP+EY\n5Zz856rS/VTiWIVjpODtOYx0zh0ys5eAuArbHj/+icdeg7fUdWdgxymOJ3JK6mFIKPkcmGJmbcE7\nh7Nv+SK8lTYBbgC+qsnOnXOHgcNmdryHckOF1XOA231lpTGz3r6qwFXZBHQ0s5G+7ePNLKoG+wFo\ngTeB5JlZe7y9Bn+sBH4CzDKzTn5+RqRK6mFIyHDOrTOzPwILzKwc7xfiVOBO4F9m9msgF7ipFoe5\nCXjRzBzwaYXlL+C93LPCN6idy3+mAK0s1hIzuwb4m2+wuRDvOEa19uPb1yozWwlsxDtD2tf+NsY5\nt9A3oP+RmZ13/HKeSE2oWq2IiPhFl6RERMQvShgiIuIXJQwREfGLEoaIiPhFCUNERPyihCEiIn5R\nwhAREb8oYYiIiF/+P+V9sJa993A7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c19ff8438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sigma = 40.\n",
    "kernel_size = int(math.floor(3.5*sigma))\n",
    "kernel = np.exp(-np.power(np.arange(2*kernel_size+1)-kernel_size, 2.)/(2*np.power(sigma, 2.)))\n",
    "kernel /= kernel.sum()\n",
    "\n",
    "smooth_error = custom_convolution(sorted_error, kernel)\n",
    "plt.figure()\n",
    "plt.plot(smooth_error)\n",
    "plt.xlabel('confidence rank')\n",
    "plt.ylabel('% error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump((model_cnn, model_lr, model_ensemble), open(models_filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
